List of usage examples for java.lang Thread interrupt
public void interrupt()
From source file:org.sakaiproject.citation.tool.CitationHelperAction.java
/** * * @param data/* w w w.j a v a 2s. co m*/ */ public void doCancelSearch(RunData data) { // get state and params SessionState state = ((JetspeedRunData) data).getPortletSessionState(((JetspeedRunData) data).getJs_peid()); ParameterParser params = data.getParameters(); int requestStateId = params.getInt("requestStateId", 0); restoreRequestState(state, new String[] { CitationHelper.RESOURCES_REQUEST_PREFIX, CitationHelper.CITATION_PREFIX }, requestStateId); // cancel the running search ActiveSearch search = (ActiveSearch) state.getAttribute(STATE_SEARCH_INFO); if (search != null) { Thread searchThread = search.getSearchThread(); if (searchThread != null) { try { searchThread.interrupt(); } catch (SecurityException se) { // not able to interrupt search logger.warn("doSearch() [in ThreadGroup " + Thread.currentThread().getThreadGroup().getName() + "] unable to interrupt search Thread [name=" + searchThread.getName() + ", id=" + searchThread.getId() + ", group=" + searchThread.getThreadGroup().getName() + "]"); } } } }
From source file:gui.images.ImageHubExplorer.java
/** * This method sets the image of the specified history index as the * currently selected image and updates all the views. * * @param index Integer that is the history index of the image to select as * the current image./*from ww w .jav a2 s. c o m*/ */ private synchronized void setSelectedImageForHistoryIndex(int historyIndex) { // Update the selected image panels. BufferedImage photo = getPhoto(selectedImageHistory.get(historyIndex)); selectedImagePanelClassNeighborMain.setImage(photo); selectedImagePanelClassNeighbor.setImage(photo); selectedImagePanelClass.setImage(photo); selectedImagePanelSearch.setImage(photo); int index = selectedImageHistory.get(historyIndex); // Update the labels with the new name. String shortPath = imgPaths.get(index).substring(workspace.getPath().length(), imgPaths.get(index).length()); selectedImagePathLabelClassNeighborMain.setText(shortPath); selectedImagePathLabelClassNeighbor.setText(shortPath); selectedImagePathLabelClass.setText(shortPath); selectedImagePathLabelSearch.setText(shortPath); // Update the class colors. selectedImageLabelClassNeighborMain.setBackground(classColors[quantizedRepresentation.getLabelOf(index)]); selectedImageLabelClassNeighbor.setBackground(classColors[quantizedRepresentation.getLabelOf(index)]); selectedImageLabelClass.setBackground(classColors[quantizedRepresentation.getLabelOf(index)]); selectedImageLabelSearch.setBackground(classColors[quantizedRepresentation.getLabelOf(index)]); // Refresh the display. selectedImageLabelClassNeighborMain.setOpaque(true); selectedImageLabelClassNeighbor.setOpaque(true); selectedImageLabelClass.setOpaque(true); selectedImageLabelSearch.setOpaque(true); selectedImageLabelClassNeighborMain.repaint(); selectedImageLabelClassNeighbor.repaint(); selectedImageLabelClass.repaint(); selectedImageLabelSearch.repaint(); // Update the nearest neighbors and the reverse nearest neighbors. NeighborSetFinder nsf = getNSF(); nnPanel.removeAll(); rnnPanel.removeAll(); nnPanel.revalidate(); nnPanel.repaint(); rnnPanel.revalidate(); rnnPanel.repaint(); int[][] kneighbors = nsf.getKNeighbors(); for (int neighborIndex = 0; neighborIndex < neighborhoodSize; neighborIndex++) { BufferedImage thumb = thumbnails.get(kneighbors[index][neighborIndex]); try { Thread t = new Thread(new SetImageNeighborsHelper(nnPanel, thumb, quantizedRepresentation.getLabelOf(kneighbors[index][neighborIndex]), kneighbors[index][neighborIndex])); t.start(); t.join(500); if (t.isAlive()) { t.interrupt(); } } catch (Throwable thr) { System.err.println(thr.getMessage()); } } ArrayList<Integer>[] rrns = rnnSetsAllK[neighborhoodSize - 1]; if (rrns[index] != null && rrns[index].size() > 0) { for (int i = 0; i < rrns[index].size(); i++) { BufferedImage thumb = thumbnails.get(rrns[index].get(i)); try { Thread t = new Thread(new SetImageNeighborsHelper(rnnPanel, thumb, quantizedRepresentation.getLabelOf(rrns[index].get(i)), rrns[index].get(i))); t.start(); t.join(500); if (t.isAlive()) { t.interrupt(); } } catch (Throwable thr) { System.err.println(thr.getMessage()); } } } // Refresh the neighbor and reverse neighbor panels. nnPanel.revalidate(); nnPanel.repaint(); rnnPanel.revalidate(); rnnPanel.repaint(); // Visualize the neighbor occurrence profile of the selected image. DefaultPieDataset pieData = new DefaultPieDataset(); for (int c = 0; c < numClasses; c++) { pieData.setValue(classNames[c], occurrenceProfilesAllK[neighborhoodSize - 1][index][c]); } JFreeChart chart = ChartFactory.createPieChart3D("occurrence profile", pieData, true, true, false); PiePlot3D plot = (PiePlot3D) chart.getPlot(); plot.setStartAngle(290); plot.setDirection(Rotation.CLOCKWISE); plot.setForegroundAlpha(0.5f); PieRenderer prend = new PieRenderer(classColors); prend.setColor(plot, pieData); ChartPanel chartPanel = new ChartPanel(chart); chartPanel.setPreferredSize(new Dimension(240, 200)); occProfileChartHolder.removeAll(); occProfileChartHolder.add(chartPanel); occProfileChartHolder.revalidate(); occProfileChartHolder.repaint(); }
From source file:gui.images.ImageHubExplorer.java
/** * This method sets the image of the specified index as the currently * selected image and updates all the views. * * @param index Integer that is the index of the image to select as the * current image.//from w w w .j a v a2 s . co m */ private synchronized void setSelectedImageForIndex(int index) { try { // Update the selected image panels. BufferedImage photo = getPhoto(index); selectedImagePanelClassNeighborMain.setImage(photo); selectedImagePanelClassNeighbor.setImage(photo); selectedImagePanelClass.setImage(photo); selectedImagePanelSearch.setImage(photo); String shortPath = imgPaths.get(index).substring(workspace.getPath().length(), imgPaths.get(index).length()); // Update the labels with the new name. selectedImagePathLabelClassNeighborMain.setText(shortPath); selectedImagePathLabelClassNeighbor.setText(shortPath); selectedImagePathLabelClass.setText(shortPath); selectedImagePathLabelSearch.setText(shortPath); // Update the class colors. selectedImageLabelClassNeighborMain .setBackground(classColors[quantizedRepresentation.getLabelOf(index)]); selectedImageLabelClassNeighbor.setBackground(classColors[quantizedRepresentation.getLabelOf(index)]); selectedImageLabelClass.setBackground(classColors[quantizedRepresentation.getLabelOf(index)]); selectedImageLabelSearch.setBackground(classColors[quantizedRepresentation.getLabelOf(index)]); // Refresh the display. selectedImageLabelClassNeighborMain.setOpaque(true); selectedImageLabelClassNeighbor.setOpaque(true); selectedImageLabelClass.setOpaque(true); selectedImageLabelSearch.setOpaque(true); selectedImageLabelClassNeighborMain.repaint(); selectedImageLabelClassNeighbor.repaint(); selectedImageLabelClass.repaint(); selectedImageLabelSearch.repaint(); // Update the history. if (selectedImageHistory == null) { selectedImageHistory = new ArrayList<>(200); selectedImageIndexInHistory = -1; } // Discard the future history. if (selectedImageIndexInHistory < selectedImageHistory.size() - 1) { for (int i = selectedImageHistory.size() - 1; i > selectedImageIndexInHistory; i--) { selectedImageHistory.remove(i); } } selectedImageHistory.add(index); selectedImageIndexInHistory = selectedImageHistory.size() - 1; // Update the nearest neighbors and the reverse nearest neighbors. NeighborSetFinder nsf = getNSF(); nnPanel.removeAll(); rnnPanel.removeAll(); nnPanel.revalidate(); nnPanel.repaint(); rnnPanel.revalidate(); rnnPanel.repaint(); int[][] kneighbors = nsf.getKNeighbors(); for (int neighborIndex = 0; neighborIndex < neighborhoodSize; neighborIndex++) { // Insert all the nearest neighbors to their panel. BufferedImage thumb = thumbnails.get(kneighbors[index][neighborIndex]); try { Thread t = new Thread(new SetImageNeighborsHelper(nnPanel, thumb, quantizedRepresentation.getLabelOf(kneighbors[index][neighborIndex]), kneighbors[index][neighborIndex])); t.start(); t.join(500); if (t.isAlive()) { t.interrupt(); } } catch (Throwable thr) { System.err.println(thr.getMessage()); } } // Insert all the reverse nearest neighbors to their panel. ArrayList<Integer>[] rrns = null; if (rnnSetsAllK != null) { rrns = rnnSetsAllK[neighborhoodSize - 1]; } if (rrns != null && rrns[index] != null && rrns[index].size() > 0) { for (int i = 0; i < rrns[index].size(); i++) { BufferedImage thumb = thumbnails.get(rrns[index].get(i)); try { Thread t = new Thread(new SetImageNeighborsHelper(rnnPanel, thumb, quantizedRepresentation.getLabelOf(rrns[index].get(i)), rrns[index].get(i))); t.start(); t.join(500); if (t.isAlive()) { t.interrupt(); } } catch (Throwable thr) { System.err.println(thr.getMessage()); } } } // Refresh the neighbor and reverse neighbor panels. nnPanel.revalidate(); nnPanel.repaint(); rnnPanel.revalidate(); rnnPanel.repaint(); // Visualize the neighbor occurrence profile of the selected image. DefaultPieDataset pieData = new DefaultPieDataset(); for (int c = 0; c < numClasses; c++) { pieData.setValue(classNames[c], occurrenceProfilesAllK[neighborhoodSize - 1][index][c]); } JFreeChart chart = ChartFactory.createPieChart3D("occurrence " + "profile", pieData, true, true, false); PiePlot3D plot = (PiePlot3D) chart.getPlot(); plot.setStartAngle(290); plot.setDirection(Rotation.CLOCKWISE); plot.setForegroundAlpha(0.5f); PieRenderer prend = new PieRenderer(classColors); prend.setColor(plot, pieData); ChartPanel chartPanel = new ChartPanel(chart); chartPanel.setPreferredSize(new Dimension(240, 200)); occProfileChartHolder.removeAll(); occProfileChartHolder.add(chartPanel); occProfileChartHolder.revalidate(); occProfileChartHolder.repaint(); } catch (Exception e) { System.err.println(e.getMessage()); } }
From source file:org.languagetool.rules.spelling.suggestions.SuggestionChangesTest.java
public void testChanges() throws IOException, InterruptedException { File configFile = new File(System.getProperty("config", "SuggestionChangesTestConfig.json")); ObjectMapper mapper = new ObjectMapper(new JsonFactory().enable(JsonParser.Feature.ALLOW_COMMENTS)); SuggestionChangesTestConfig config = mapper.readValue(configFile, SuggestionChangesTestConfig.class); SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd_HH:mm:ss"); String timestamp = dateFormat.format(new Date()); Path loggingFile = Paths.get(config.logDir, String.format("suggestionChangesExperiment_%s.log", timestamp)); Path datasetFile = Paths.get(config.logDir, String.format("suggestionChangesExperiment_%s.csv", timestamp)); BufferedWriter writer = Files.newBufferedWriter(loggingFile); CSVPrinter datasetWriter = new CSVPrinter(Files.newBufferedWriter(datasetFile), CSVFormat.DEFAULT.withEscape('\\')); List<String> datasetHeader = new ArrayList<>( Arrays.asList("sentence", "correction", "covered", "replacement", "dataset_id")); SuggestionsChanges.init(config, writer); writer.write("Evaluation configuration: \n"); String configContent = String.join("\n", Files.readAllLines(configFile.toPath())); writer.write(configContent);//w ww . j ava 2s . c om writer.write("\nRunning experiments: \n"); int experimentId = 0; for (SuggestionChangesExperiment experiment : SuggestionsChanges.getInstance().getExperiments()) { experimentId++; writer.write(String.format("#%d: %s%n", experimentId, experiment)); datasetHeader.add(String.format("experiment_%d_suggestions", experimentId)); datasetHeader.add(String.format("experiment_%d_metadata", experimentId)); datasetHeader.add(String.format("experiment_%d_suggestions_metadata", experimentId)); } writer.newLine(); datasetWriter.printRecord(datasetHeader); BlockingQueue<SuggestionTestData> tasks = new LinkedBlockingQueue<>(1000); ConcurrentLinkedQueue<Pair<SuggestionTestResultData, String>> results = new ConcurrentLinkedQueue<>(); List<SuggestionTestThread> threads = new ArrayList<>(); for (int i = 0; i < Runtime.getRuntime().availableProcessors(); i++) { SuggestionTestThread worker = new SuggestionTestThread(tasks, results); worker.start(); threads.add(worker); } // Thread for writing results from worker threads into CSV Thread logger = new Thread(() -> { try { long messages = 0; //noinspection InfiniteLoopStatement while (true) { Pair<SuggestionTestResultData, String> message = results.poll(); if (message != null) { writer.write(message.getRight()); SuggestionTestResultData result = message.getLeft(); int datasetId = 1 + config.datasets.indexOf(result.getInput().getDataset()); if (result != null && result.getSuggestions() != null && !result.getSuggestions().isEmpty() && result.getSuggestions().stream() .noneMatch(m -> m.getSuggestedReplacements() == null || m.getSuggestedReplacements().isEmpty())) { List<Object> record = new ArrayList<>(Arrays.asList(result.getInput().getSentence(), result.getInput().getCorrection(), result.getInput().getCovered(), result.getInput().getReplacement(), datasetId)); for (RuleMatch match : result.getSuggestions()) { List<String> suggestions = match.getSuggestedReplacements(); record.add(mapper.writeValueAsString(suggestions)); // features extracted by SuggestionsOrdererFeatureExtractor record.add(mapper.writeValueAsString(match.getFeatures())); List<SortedMap<String, Float>> suggestionsMetadata = new ArrayList<>(); for (SuggestedReplacement replacement : match.getSuggestedReplacementObjects()) { suggestionsMetadata.add(replacement.getFeatures()); } record.add(mapper.writeValueAsString(suggestionsMetadata)); } datasetWriter.printRecord(record); } if (++messages % 1000 == 0) { writer.flush(); System.out.printf("Evaluated %d corrections.%n", messages); } } } } catch (IOException e) { throw new RuntimeException(e); } }); logger.setDaemon(true); logger.start(); // format straight from database dump String[] header = { "id", "sentence", "correction", "language", "rule_id", "suggestion_pos", "accept_language", "country", "region", "created_at", "updated_at", "covered", "replacement", "text_session_id", "client" }; int datasetId = 0; // read data, send to worker threads via queue for (SuggestionChangesDataset dataset : config.datasets) { writer.write(String.format("Evaluating dataset #%d: %s.%n", ++datasetId, dataset)); CSVFormat format = CSVFormat.DEFAULT; if (dataset.type.equals("dump")) { format = format.withEscape('\\').withNullString("\\N").withHeader(header); } else if (dataset.type.equals("artificial")) { format = format.withEscape('\\').withFirstRecordAsHeader(); } try (CSVParser parser = new CSVParser(new FileReader(dataset.path), format)) { for (CSVRecord record : parser) { String lang = record.get("language"); String rule = dataset.type.equals("dump") ? record.get("rule_id") : ""; String covered = record.get("covered"); String replacement = record.get("replacement"); String sentence = record.get("sentence"); String correction = record.isSet("correction") ? record.get("correction") : ""; String acceptLanguage = dataset.type.equals("dump") ? record.get("accept_language") : ""; if (sentence == null || sentence.trim().isEmpty()) { continue; } if (!config.language.equals(lang)) { continue; // TODO handle auto maybe? } if (dataset.type.equals("dump") && !config.rule.equals(rule)) { continue; } // correction column missing in export from doccano; workaround if (dataset.enforceCorrect && !record.isSet("correction")) { throw new IllegalStateException("enforceCorrect in dataset configuration enabled," + " but column 'correction' is not set for entry " + record); } if (dataset.type.equals("dump") && dataset.enforceAcceptLanguage) { if (acceptLanguage != null) { String[] entries = acceptLanguage.split(",", 2); if (entries.length == 2) { String userLanguage = entries[0]; // TODO: what to do with e.g. de-AT,de-DE;... if (!config.language.equals(userLanguage)) { continue; } } } } tasks.put(new SuggestionTestData(lang, sentence, covered, replacement, correction, dataset)); } } } for (Thread t : threads) { t.join(); } logger.join(10000L); logger.interrupt(); datasetWriter.close(); }
From source file:fm.smart.r1.ItemActivity.java
public void onActivityResult(final int requestCode, final int resultCode, final Intent data) { super.onActivityResult(requestCode, resultCode, data); // this should be called once image has been chosen by user // using requestCode to pass item id - haven't worked out any other way // to do it// w w w. j a v a 2s . c o m // if (requestCode == SELECT_IMAGE) if (resultCode == Activity.RESULT_OK) { // TODO check if user is logged in if (LoginActivity.isNotLoggedIn(this)) { Intent intent = new Intent(Intent.ACTION_VIEW); intent.setClassName(this, LoginActivity.class.getName()); intent.setFlags(Intent.FLAG_ACTIVITY_NO_HISTORY); // avoid navigation back to this? LoginActivity.return_to = ItemActivity.class.getName(); LoginActivity.params = new HashMap<String, String>(); LoginActivity.params.put("item_id", (String) item.getId()); startActivity(intent); // TODO in this case forcing the user to rechoose the image // seems a little // rude - should probably auto-submit here ... } else { // Bundle extras = data.getExtras(); // String sentence_id = (String) extras.get("sentence_id"); final ProgressDialog myOtherProgressDialog = new ProgressDialog(this); myOtherProgressDialog.setTitle("Please Wait ..."); myOtherProgressDialog.setMessage("Uploading image ..."); myOtherProgressDialog.setIndeterminate(true); myOtherProgressDialog.setCancelable(true); final Thread add_image = new Thread() { public void run() { // TODO needs to check for interruptibility String sentence_id = Integer.toString(requestCode); Uri selectedImage = data.getData(); // Bitmap bitmap = Media.getBitmap(getContentResolver(), // selectedImage); // ByteArrayOutputStream bytes = new // ByteArrayOutputStream(); // bitmap.compress(Bitmap.CompressFormat.JPEG, 40, // bytes); // ByteArrayInputStream fileInputStream = new // ByteArrayInputStream( // bytes.toByteArray()); // TODO Might have to save to file system first to get // this // to work, // argh! // could think of it as saving to cache ... // add image to sentence FileInputStream is = null; FileOutputStream os = null; File file = null; ContentResolver resolver = getContentResolver(); try { Bitmap bitmap = Media.getBitmap(getContentResolver(), selectedImage); ByteArrayOutputStream bytes = new ByteArrayOutputStream(); bitmap.compress(Bitmap.CompressFormat.JPEG, 40, bytes); // ByteArrayInputStream bais = new // ByteArrayInputStream(bytes.toByteArray()); // FileDescriptor fd = // resolver.openFileDescriptor(selectedImage, // "r").getFileDescriptor(); // is = new FileInputStream(fd); String filename = "test.jpg"; File dir = ItemActivity.this.getDir("images", MODE_WORLD_READABLE); file = new File(dir, filename); os = new FileOutputStream(file); // while (bais.available() > 0) { // / os.write(bais.read()); // } os.write(bytes.toByteArray()); os.close(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } finally { if (os != null) { try { os.close(); } catch (IOException e) { } } if (is != null) { try { is.close(); } catch (IOException e) { } } } // File file = new // File(Uri.decode(selectedImage.toString())); // ensure item is in users default list ItemActivity.add_item_result = new AddItemResult(Main.lookup.addItemToGoal(Main.transport, Main.default_study_goal_id, item.getId(), null)); Result result = ItemActivity.add_item_result; if (ItemActivity.add_item_result.success() || ItemActivity.add_item_result.alreadyInList()) { // ensure sentence is in users default goal ItemActivity.add_sentence_goal_result = new AddSentenceResult( Main.lookup.addSentenceToGoal(Main.transport, Main.default_study_goal_id, item.getId(), sentence_id, null)); result = ItemActivity.add_sentence_goal_result; if (ItemActivity.add_sentence_goal_result.success()) { String media_entity = "http://test.com/test.jpg"; String author = "tansaku"; String author_url = "http://smart.fm/users/tansaku"; Log.d("DEBUG-IMAGE-URI", selectedImage.toString()); ItemActivity.add_image_result = addImage(file, media_entity, author, author_url, "1", sentence_id, (String) item.getId(), Main.default_study_goal_id); result = ItemActivity.add_image_result; } } final Result display = result; myOtherProgressDialog.dismiss(); ItemActivity.this.runOnUiThread(new Thread() { public void run() { final AlertDialog dialog = new AlertDialog.Builder(ItemActivity.this).create(); dialog.setTitle(display.getTitle()); dialog.setMessage(display.getMessage()); dialog.setButton("OK", new DialogInterface.OnClickListener() { public void onClick(DialogInterface dialog, int which) { if (ItemActivity.add_image_result != null && ItemActivity.add_image_result.success()) { ItemListActivity.loadItem(ItemActivity.this, item.getId().toString()); } } }); dialog.show(); } }); } }; myOtherProgressDialog.setButton("Cancel", new DialogInterface.OnClickListener() { public void onClick(DialogInterface dialog, int which) { add_image.interrupt(); } }); OnCancelListener ocl = new OnCancelListener() { public void onCancel(DialogInterface arg0) { add_image.interrupt(); } }; myOtherProgressDialog.setOnCancelListener(ocl); closeMenu(); myOtherProgressDialog.show(); add_image.start(); } } }
From source file:fm.smart.r1.activity.ItemActivity.java
@Override public void onActivityResult(final int requestCode, final int resultCode, final Intent data) { super.onActivityResult(requestCode, resultCode, data); // this should be called once image has been chosen by user // using requestCode to pass item id - haven't worked out any other way // to do it// www .j a va 2 s. co m // if (requestCode == SELECT_IMAGE) if (resultCode == Activity.RESULT_OK) { // TODO check if user is logged in if (Main.isNotLoggedIn(this)) { Intent intent = new Intent(Intent.ACTION_VIEW); intent.setClassName(this, LoginActivity.class.getName()); intent.setFlags(Intent.FLAG_ACTIVITY_NO_HISTORY); // avoid navigation back to this? LoginActivity.return_to = ItemActivity.class.getName(); LoginActivity.params = new HashMap<String, String>(); LoginActivity.params.put("item_id", (String) item.item_node.atts.get("id")); startActivity(intent); // TODO in this case forcing the user to rechoose the image // seems a little // rude - should probably auto-submit here ... } else { // Bundle extras = data.getExtras(); // String sentence_id = (String) extras.get("sentence_id"); final ProgressDialog myOtherProgressDialog = new ProgressDialog(this); myOtherProgressDialog.setTitle("Please Wait ..."); myOtherProgressDialog.setMessage("Uploading image ..."); myOtherProgressDialog.setIndeterminate(true); myOtherProgressDialog.setCancelable(true); final Thread add_image = new Thread() { public void run() { // TODO needs to check for interruptibility String sentence_id = Integer.toString(requestCode); Uri selectedImage = data.getData(); // Bitmap bitmap = Media.getBitmap(getContentResolver(), // selectedImage); // ByteArrayOutputStream bytes = new // ByteArrayOutputStream(); // bitmap.compress(Bitmap.CompressFormat.JPEG, 40, // bytes); // ByteArrayInputStream fileInputStream = new // ByteArrayInputStream( // bytes.toByteArray()); // TODO Might have to save to file system first to get // this // to work, // argh! // could think of it as saving to cache ... // add image to sentence FileInputStream is = null; FileOutputStream os = null; File file = null; ContentResolver resolver = getContentResolver(); try { Bitmap bitmap = Media.getBitmap(getContentResolver(), selectedImage); ByteArrayOutputStream bytes = new ByteArrayOutputStream(); bitmap.compress(Bitmap.CompressFormat.JPEG, 40, bytes); // ByteArrayInputStream bais = new // ByteArrayInputStream(bytes.toByteArray()); // FileDescriptor fd = // resolver.openFileDescriptor(selectedImage, // "r").getFileDescriptor(); // is = new FileInputStream(fd); String filename = "test.jpg"; File dir = ItemActivity.this.getDir("images", MODE_WORLD_READABLE); file = new File(dir, filename); os = new FileOutputStream(file); // while (bais.available() > 0) { // / os.write(bais.read()); // } os.write(bytes.toByteArray()); os.close(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } finally { if (os != null) { try { os.close(); } catch (IOException e) { } } if (is != null) { try { is.close(); } catch (IOException e) { } } } // File file = new // File(Uri.decode(selectedImage.toString())); // ensure item is in users default list ItemActivity.add_item_result = addItemToList(Main.default_study_list_id, (String) item.item_node.atts.get("id"), ItemActivity.this); Result result = ItemActivity.add_item_result; if (ItemActivity.add_item_result.success() || ItemActivity.add_item_result.alreadyInList()) { // ensure sentence is in users default list ItemActivity.add_sentence_list_result = addSentenceToList(sentence_id, (String) item.item_node.atts.get("id"), Main.default_study_list_id, ItemActivity.this); result = ItemActivity.add_sentence_list_result; if (ItemActivity.add_sentence_list_result.success()) { String media_entity = "http://test.com/test.jpg"; String author = "tansaku"; String author_url = "http://smart.fm/users/tansaku"; Log.d("DEBUG-IMAGE-URI", selectedImage.toString()); ItemActivity.add_image_result = addImage(file, media_entity, author, author_url, "1", sentence_id, (String) item.item_node.atts.get("id"), Main.default_study_list_id); result = ItemActivity.add_image_result; } } final Result display = result; myOtherProgressDialog.dismiss(); ItemActivity.this.runOnUiThread(new Thread() { public void run() { final AlertDialog dialog = new AlertDialog.Builder(ItemActivity.this).create(); dialog.setTitle(display.getTitle()); dialog.setMessage(display.getMessage()); dialog.setButton("OK", new DialogInterface.OnClickListener() { public void onClick(DialogInterface dialog, int which) { if (ItemActivity.add_image_result != null && ItemActivity.add_image_result.success()) { ItemListActivity.loadItem(ItemActivity.this, item.item_node.atts.get("id").toString()); } } }); dialog.show(); } }); } }; myOtherProgressDialog.setButton("Cancel", new DialogInterface.OnClickListener() { public void onClick(DialogInterface dialog, int which) { add_image.interrupt(); } }); OnCancelListener ocl = new OnCancelListener() { public void onCancel(DialogInterface arg0) { add_image.interrupt(); } }; myOtherProgressDialog.setOnCancelListener(ocl); closeMenu(); myOtherProgressDialog.show(); add_image.start(); } } }
From source file:com.baifendian.swordfish.execserver.engine.hive.HiveSqlExec.java
/** * sql ? ?, ?, execute, ?//from ww w. j av a2 s . c o m * * @param createFuncs ? * @param sqls sql * @param isContinue ?, ??? * @param resultCallback , ? * @param queryLimit ? * @param remainTime ?, */ public boolean execute(List<String> createFuncs, List<String> sqls, boolean isContinue, ResultCallback resultCallback, Integer queryLimit, int remainTime) { // ? if (remainTime <= 0) { return false; } // ? queryLimit = (queryLimit != null) ? queryLimit : defaultQueryLimit; HiveConnection hiveConnection = null; Statement sta = null; Thread logThread = null; // hive ? HiveService2ConnectionInfo hiveService2ConnectionInfo = hiveUtil.getHiveService2ConnectionInfo(userName); logger.info("execution connection information:{}", hiveService2ConnectionInfo); HiveService2Client hiveService2Client = hiveUtil.getHiveService2Client(); try { try { hiveConnection = hiveService2Client.borrowClient(hiveService2ConnectionInfo); sta = hiveConnection.createStatement(); // sta.setQueryTimeout(remainTime); // logThread = new Thread(new JdbcLogRunnable(sta)); logThread.setDaemon(true); logThread.start(); // set queue if (queueSQL != null) { logger.info("hive queue : {}", queueSQL); sta.execute(queueSQL); } // function if (createFuncs != null) { for (String createFunc : createFuncs) { logger.info("hive create function sql: {}", createFunc); sta.execute(createFunc); } } } catch (Exception e) { logger.error("execute query exception", e); // , , ? handlerResults(0, sqls, FlowStatus.FAILED, resultCallback); return false; } // sql ? for (int index = 0; index < sqls.size(); ++index) { String sql = sqls.get(index); Date startTime = new Date(); logger.info("hive execute sql: {}", sql); ExecResult execResult = new ExecResult(); execResult.setIndex(index); execResult.setStm(sql); try { // ? query show ? if (HiveUtil.isTokQuery(sql) || HiveUtil.isLikeShowStm(sql)) { sta.setMaxRows(queryLimit); ResultSet res = sta.executeQuery(sql); ResultSetMetaData resultSetMetaData = res.getMetaData(); int count = resultSetMetaData.getColumnCount(); List<String> colums = new ArrayList<>(); for (int i = 1; i <= count; i++) { colums.add(resultSetMetaData.getColumnLabel( i)/*parseColumnName(resultSetMetaData.getColumnLabel(i), colums)*/); } execResult.setTitles(colums); List<List<String>> datas = new ArrayList<>(); // 1, query ? if (count > 1 || HiveUtil.isTokQuery(sql)) { while (res.next()) { List<String> values = new ArrayList<>(); for (int i = 1; i <= count; ++i) { values.add(res.getString(i)); } datas.add(values); } } else { StringBuffer buffer = new StringBuffer(); while (res.next()) { buffer.append(res.getString(1)); buffer.append("\n"); } List<String> values = new ArrayList<>(); values.add(buffer.toString().trim()); datas.add(values); } execResult.setValues(datas); } else { sta.execute(sql); } // ?? execResult.setStatus(FlowStatus.SUCCESS); // ? if (resultCallback != null) { Date endTime = new Date(); resultCallback.handleResult(execResult, startTime, endTime); } } catch (SQLTimeoutException e) { // sql logger.error("executeQuery timeout exception", e); handlerResults(index, sqls, FlowStatus.FAILED, resultCallback); return false; } catch (DaoSemanticException | HiveSQLException e) { // logger.error("executeQuery exception", e); if (isContinue) { handlerResult(index, sql, FlowStatus.FAILED, resultCallback); } else { handlerResults(index, sqls, FlowStatus.FAILED, resultCallback); return false; } } catch (Exception e) { // TTransport if (e.toString().contains("TTransportException")) { logger.error("Get TTransportException return a client", e); // ??? // hiveService2Client.invalidateObject(hiveService2ConnectionInfo, hiveConnection); handlerResults(index, sqls, FlowStatus.FAILED, resultCallback); return false; } // socket if (e.toString().contains("SocketException")) { logger.error("SocketException clear pool", e); hiveService2Client.clear(); handlerResults(index, sqls, FlowStatus.FAILED, resultCallback); return false; } logger.error("executeQuery exception", e); if (isContinue) { handlerResult(index, sql, FlowStatus.FAILED, resultCallback); } else { handlerResults(index, sqls, FlowStatus.FAILED, resultCallback); return false; } } } } finally { // try { if (sta != null) { sta.close(); } } catch (Exception e) { logger.error("Catch an exception", e); } try { // if (hiveConnection != null) { // hiveConnection.close(); // , ?? hiveService2Client.returnClient(hiveService2ConnectionInfo, hiveConnection); } } catch (Exception e) { logger.error("Catch an exception", e); } // try { if (logThread != null) { logThread.interrupt(); logThread.join(HiveUtil.DEFAULT_QUERY_PROGRESS_THREAD_TIMEOUT); } } catch (Exception e) { // logger.error("Catch an exception", e); } } return true; }
From source file:fr.msch.wissl.server.Library.java
private Library() { this.songs = new ConcurrentLinkedQueue<Song>(); this.toRead = new ConcurrentHashMap<String, File>(); this.files = new ConcurrentLinkedQueue<File>(); this.toInsert = new ConcurrentLinkedQueue<Song>(); this.hashes = new HashSet<String>(); this.artworks = new HashMap<String, Map<String, String>>(); this.artworkFallback = new FileFilter() { @Override// w w w .ja v a 2 s. co m public boolean accept(File pathname) { return Pattern.matches(".*[.](jpeg|jpg|png|bmp|gif)$", pathname.getName().toLowerCase()); } }; Runnable timer = new Runnable() { @Override public void run() { while (!kill) { final long t1 = System.currentTimeMillis(); final List<File> music = new ArrayList<File>(); for (String path : Config.getMusicPath()) { music.add(new File(path)); } addSongCount = 0; skipSongCount = 0; failedSongCount = 0; fileSearchTime = 0; dbCheckTime = 0; fileReadTime = 0; dbInsertTime = 0; resizeTime = 0; songs.clear(); toRead.clear(); files.clear(); hashes.clear(); toInsert.clear(); artworks.clear(); songsTodo = 0; songsDone = 0; working = true; stop = false; percentDone = 0.0f; secondsLeft = -1; artworkRegex = Pattern.compile(Config.getArtworkRegex()); artworkFilter = new FileFilter() { @Override public boolean accept(File pathname) { return (artworkRegex.matcher(pathname.getName().toLowerCase()).matches()); } }; // walks filesystem and indexes files that look like music fileSearchDone = false; Thread fileSearch = new Thread(new Runnable() { public void run() { long f1 = System.currentTimeMillis(); for (File f : music) { try { listFiles(f, files); } catch (IOException e) { Logger.error("Failed to add directory to library: " + f.getAbsolutePath(), e); } catch (InterruptedException e) { return; } } fileSearchDone = true; fileSearchTime = (System.currentTimeMillis() - f1); } }); fileSearch.start(); // exclude files that are already in DB dbCheckDone = false; Thread dbCheck = new Thread(new Runnable() { public void run() { while (!stop && !dbCheckDone) { long f1 = System.currentTimeMillis(); while (!files.isEmpty()) { File f = files.remove(); String hash = new String(md5.digest(f.getAbsolutePath().getBytes())); boolean hasSong = false; try { hasSong = DB.get().hasSong(hash); } catch (SQLException e) { Logger.error("Failed to query DB for file " + f.getAbsolutePath(), e); } if (!hasSong) { toRead.put(hash, f); } else { skipSongCount++; } hashes.add(hash); } dbCheckTime += (System.currentTimeMillis() - f1); if (fileSearchDone && files.isEmpty()) { dbCheckDone = true; return; } } } }); dbCheck.start(); // read file metadata fileReadDone = false; Thread fileRead = new Thread(new Runnable() { public void run() { while (!stop && !fileReadDone) { long f1 = System.currentTimeMillis(); Iterator<Entry<String, File>> it = toRead.entrySet().iterator(); while (it.hasNext()) { Entry<String, File> f = it.next(); it.remove(); try { Song s = getSong(f.getValue(), f.getKey()); songs.add(s); addSongCount++; } catch (IOException e) { Logger.warn("Failed to read music file " + f.getValue(), e); failedSongCount++; } } fileReadTime += (System.currentTimeMillis() - f1); if (dbCheckDone && toRead.isEmpty()) { fileReadDone = true; return; } } } }); fileRead.start(); // resize images resizeDone = false; Thread resize = new Thread(new Runnable() { public void run() { while (!stop && !resizeDone) { long f1 = System.currentTimeMillis(); while (!songs.isEmpty()) { Song s = songs.remove(); String path = null; Map<String, String> m = artworks.get(s.artist.name); if (m != null && m.containsKey(s.album.name)) { path = m.get(s.album.name); } if (path != null) { if (new File(path + "_SCALED.jpg").exists()) { path = path + "_SCALED.jpg"; } else { try { path = resizeArtwork(path); } catch (IOException e) { Logger.warn("Failed to resize image", e); } } s.album.artwork_path = path; s.album.artwork_id = "" + System.currentTimeMillis(); } toInsert.add(s); } resizeTime += (System.currentTimeMillis() - f1); if (fileReadDone && songs.isEmpty()) { resizeDone = true; return; } } } }); resize.start(); // insert Songs in DB Thread dbInsert = new Thread(new Runnable() { public void run() { while (!stop) { long f1 = System.currentTimeMillis(); while (!toInsert.isEmpty()) { Song s = toInsert.remove(); try { DB.get().addSong(s); } catch (SQLException e) { Logger.warn("Failed to insert in DB " + s.filepath, e); failedSongCount++; } songsDone++; percentDone = songsDone / ((float) songsTodo); float songsPerSec = songsDone / ((System.currentTimeMillis() - t1) / 1000f); secondsLeft = (long) ((songsTodo - songsDone) / songsPerSec); } dbInsertTime += (System.currentTimeMillis() - f1); if (resizeDone && toInsert.isEmpty()) { return; } } } }); dbInsert.start(); try { dbInsert.join(); } catch (InterruptedException e3) { Logger.warn("Library indexer interrupted", e3); fileSearch.interrupt(); dbCheck.interrupt(); fileRead.interrupt(); resize.interrupt(); dbInsert.interrupt(); } if (Thread.interrupted()) { Logger.warn("Library indexer has been interrupted"); continue; } // remove files from DB that were not found int removed = 0; long r1 = System.currentTimeMillis(); try { removed = DB.get().removeSongs(hashes); } catch (SQLException e3) { Logger.error("Failed to remove songs", e3); } long dbRemoveTime = (System.currentTimeMillis() - r1); // update statistics long u1 = System.currentTimeMillis(); try { DB.get().updateSongCount(); } catch (SQLException e1) { Logger.error("Failed to update song count", e1); } long dbUpdateTime = (System.currentTimeMillis() - u1); try { RuntimeStats.get().updateFromDB(); } catch (SQLException e) { Logger.error("Failed to update runtime statistics", e); } working = false; long t2 = (System.currentTimeMillis() - t1); Logger.info("Processed " + songsDone + " files " // + "(add:" + addSongCount + "," // + "skip:" + skipSongCount + "," // + "fail:" + failedSongCount + "," // + "rem:" + removed + ")"); Logger.info("Indexer took " + t2 + " (" + ((float) songsDone / ((float) t2 / 1000)) + " /s) (" // + "search:" + fileSearchTime + "," // + "check:" + dbCheckTime + ","// + "read:" + fileReadTime + "," // + "resize:" + resizeTime + "," // + "insert:" + dbInsertTime + "," // + "remove:" + dbRemoveTime + "," // + "update:" + dbUpdateTime + ")"); int seconds = Config.getMusicRefreshRate(); try { Thread.sleep(seconds * 1000); } catch (InterruptedException e) { Logger.warn("Library indexer interrupted", e); } } } }; this.thread = new Thread(timer, "MusicIndexer"); }
From source file:eu.apenet.dpt.standalone.gui.batch.ConvertAndValidateActionListener.java
public void actionPerformed(ActionEvent event) { labels = dataPreparationToolGUI.getLabels(); continueLoop = true;// ww w.jav a 2 s. com dataPreparationToolGUI.disableAllBtnAndItems(); dataPreparationToolGUI.disableEditionTab(); dataPreparationToolGUI.disableRadioButtons(); dataPreparationToolGUI.disableAllBatchBtns(); dataPreparationToolGUI.getAPEPanel().setFilename(""); final Object[] objects = dataPreparationToolGUI.getXmlEadList().getSelectedValues(); final ApexActionListener apexActionListener = this; new Thread(new Runnable() { public void run() { FileInstance uniqueFileInstance = null; String uniqueXslMessage = ""; int numberOfFiles = objects.length; int currentFileNumberBatch = 0; ProgressFrame progressFrame = new ProgressFrame(labels, parent, true, false, apexActionListener); JProgressBar batchProgressBar = progressFrame.getProgressBarBatch(); dataPreparationToolGUI.getAPEPanel().getApeTabbedPane().disableConversionBtn(); dataPreparationToolGUI.getAPEPanel().getApeTabbedPane().disableValidationBtn(); dataPreparationToolGUI.getAPEPanel().getApeTabbedPane().disableConvertAndValidateBtn(); dataPreparationToolGUI.getXmlEadList().setEnabled(false); for (Object oneFile : objects) { if (!continueLoop) { break; } File file = (File) oneFile; FileInstance fileInstance = dataPreparationToolGUI.getFileInstances().get(file.getName()); if (numberOfFiles == 1) { uniqueFileInstance = fileInstance; } if (!fileInstance.isXml()) { fileInstance.setXml(XmlChecker.isXmlParseable(file) == null); if (!fileInstance.isXml()) { if (type == CONVERT || type == CONVERT_AND_VALIDATE) { fileInstance.setConversionErrors(labels.getString("conversion.error.fileNotXml")); } else if (type == VALIDATE || type == CONVERT_AND_VALIDATE) { fileInstance.setValidationErrors(labels.getString("validation.error.fileNotXml")); } dataPreparationToolGUI.enableSaveBtn(); dataPreparationToolGUI.enableRadioButtons(); dataPreparationToolGUI.enableEditionTab(); } } SummaryWorking summaryWorking = new SummaryWorking(dataPreparationToolGUI.getResultArea(), batchProgressBar); summaryWorking.setTotalNumberFiles(numberOfFiles); summaryWorking.setCurrentFileNumberBatch(currentFileNumberBatch); Thread threadRunner = new Thread(summaryWorking); threadRunner.setName(SummaryWorking.class.toString()); threadRunner.start(); JProgressBar progressBar = null; Thread threadProgress = null; CounterThread counterThread = null; CounterCLevelCall counterCLevelCall = null; if (fileInstance.isXml()) { currentFileNumberBatch = currentFileNumberBatch + 1; if (type == CONVERT || type == CONVERT_AND_VALIDATE) { dataPreparationToolGUI.setResultAreaText(labels.getString("converting") + " " + file.getName() + " (" + (currentFileNumberBatch) + "/" + numberOfFiles + ")"); String eadid = ""; boolean doTransformation = true; if (fileInstance.getValidationSchema() .equals(Utilities.getXsdObjectFromPath(Xsd_enum.XSD_APE_SCHEMA.getPath())) || fileInstance.getValidationSchema().equals( Utilities.getXsdObjectFromPath(Xsd_enum.XSD_EAD_SCHEMA.getPath()))) { StaxTransformationTool staxTransformationTool = new StaxTransformationTool(file); staxTransformationTool.run(); LOG.debug("file has eadid? " + staxTransformationTool.isFileWithEadid()); if (!staxTransformationTool.isFileWithEadid()) { EadidQueryComponent eadidQueryComponent; if (staxTransformationTool.getUnitid() != null && !staxTransformationTool.getUnitid().equals("")) { eadidQueryComponent = new EadidQueryComponent( staxTransformationTool.getUnitid()); } else { eadidQueryComponent = new EadidQueryComponent(labels); } int result = JOptionPane.showConfirmDialog(parent, eadidQueryComponent.getMainPanel(), labels.getString("enterEADID"), JOptionPane.OK_CANCEL_OPTION); while (StringUtils.isEmpty(eadidQueryComponent.getEntryEadid()) && result != JOptionPane.CANCEL_OPTION) { result = JOptionPane.showConfirmDialog(parent, eadidQueryComponent.getMainPanel(), labels.getString("enterEADID"), JOptionPane.OK_CANCEL_OPTION); } if (result == JOptionPane.OK_OPTION) { eadid = eadidQueryComponent.getEntryEadid(); } else if (result == JOptionPane.CANCEL_OPTION) { doTransformation = false; } } } if (doTransformation) { int counterMax = 0; if (fileInstance.getConversionScriptName() .equals(Utilities.XSL_DEFAULT_APEEAD_NAME)) { progressBar = progressFrame.getProgressBarSingle(); progressBar.setVisible(true); progressFrame .setTitle(labels.getString("progressTrans") + " - " + file.getName()); CountCLevels countCLevels = new CountCLevels(); counterMax = countCLevels.countOneFile(file); if (counterMax > 0) { counterCLevelCall = new CounterCLevelCall(); counterCLevelCall.initializeCounter(counterMax); counterThread = new CounterThread(counterCLevelCall, progressBar, counterMax); threadProgress = new Thread(counterThread); threadProgress.setName(CounterThread.class.toString()); threadProgress.start(); } } try { try { File xslFile = new File(fileInstance.getConversionScriptPath()); File outputFile = new File(Utilities.TEMP_DIR + "temp_" + file.getName()); outputFile.deleteOnExit(); StringWriter xslMessages; HashMap<String, String> parameters = dataPreparationToolGUI.getParams(); parameters.put("eadidmissing", eadid); CheckIsEadFile checkIsEadFile = new CheckIsEadFile(file); checkIsEadFile.run(); if (checkIsEadFile.isEadRoot()) { File outputFile_temp = new File( Utilities.TEMP_DIR + ".temp_" + file.getName()); TransformationTool.createTransformation(FileUtils.openInputStream(file), outputFile_temp, Utilities.BEFORE_XSL_FILE, null, true, true, null, true, null); xslMessages = TransformationTool.createTransformation( FileUtils.openInputStream(outputFile_temp), outputFile, xslFile, parameters, true, true, null, true, counterCLevelCall); outputFile_temp.delete(); } else { xslMessages = TransformationTool.createTransformation( FileUtils.openInputStream(file), outputFile, xslFile, parameters, true, true, null, true, null); } fileInstance.setConversionErrors(xslMessages.toString()); fileInstance .setCurrentLocation(Utilities.TEMP_DIR + "temp_" + file.getName()); fileInstance.setConverted(); fileInstance.setLastOperation(FileInstance.Operation.CONVERT); uniqueXslMessage = xslMessages.toString(); if (xslMessages.toString().equals("")) { if (fileInstance.getConversionScriptName() .equals(Utilities.XSL_DEFAULT_APEEAD_NAME)) { fileInstance.setConversionErrors( labels.getString("conversion.noExcludedElements")); } else { fileInstance.setConversionErrors( labels.getString("conversion.finished")); } } if (!continueLoop) { break; } } catch (Exception e) { fileInstance.setConversionErrors(labels.getString("conversionException") + "\r\n\r\n-------------\r\n" + e.getMessage()); throw new Exception("Error when converting " + file.getName(), e); } if (threadProgress != null) { counterThread.stop(); threadProgress.interrupt(); } if (progressBar != null) { if (counterMax > 0) { progressBar.setValue(counterMax); } progressBar.setIndeterminate(true); } } catch (Exception e) { LOG.error("Error when converting and validating", e); } finally { summaryWorking.stop(); threadRunner.interrupt(); dataPreparationToolGUI.getXmlEadListLabel().repaint(); dataPreparationToolGUI.getXmlEadList().repaint(); if (progressBar != null) { progressBar.setVisible(false); } } } if (numberOfFiles == 1) { uniqueFileInstance = fileInstance; } } if (type == VALIDATE || type == CONVERT_AND_VALIDATE) { try { try { File fileToValidate = new File(fileInstance.getCurrentLocation()); InputStream is = FileUtils.openInputStream(fileToValidate); dataPreparationToolGUI .setResultAreaText(labels.getString("validating") + " " + file.getName() + " (" + currentFileNumberBatch + "/" + numberOfFiles + ")"); XsdObject xsdObject = fileInstance.getValidationSchema(); List<SAXParseException> exceptions; if (xsdObject.getName().equals(Xsd_enum.DTD_EAD_2002.getReadableName())) { exceptions = DocumentValidation.xmlValidationAgainstDtd( fileToValidate.getAbsolutePath(), Utilities.getUrlPathXsd(xsdObject)); } else { exceptions = DocumentValidation.xmlValidation(is, Utilities.getUrlPathXsd(xsdObject), xsdObject.isXsd11()); } if (exceptions == null || exceptions.isEmpty()) { fileInstance.setValid(true); fileInstance.setValidationErrors(labels.getString("validationSuccess")); if (xsdObject.getFileType().equals(FileInstance.FileType.EAD) && xsdObject.getName().equals("apeEAD")) { XmlQualityCheckerCall xmlQualityCheckerCall = new XmlQualityCheckerCall(); InputStream is2 = FileUtils .openInputStream(new File(fileInstance.getCurrentLocation())); TransformationTool.createTransformation(is2, null, Utilities.XML_QUALITY_FILE, null, true, true, null, false, xmlQualityCheckerCall); String xmlQualityStr = createXmlQualityString(xmlQualityCheckerCall); fileInstance.setValidationErrors( fileInstance.getValidationErrors() + xmlQualityStr); fileInstance.setXmlQualityErrors( createXmlQualityErrors(xmlQualityCheckerCall)); } } else { String errors = Utilities.stringFromList(exceptions); fileInstance.setValidationErrors(errors); fileInstance.setValid(false); } fileInstance.setLastOperation(FileInstance.Operation.VALIDATE); } catch (Exception ex) { fileInstance.setValid(false); fileInstance.setValidationErrors(labels.getString("validationException") + "\r\n\r\n-------------\r\n" + ex.getMessage()); throw new Exception("Error when validating", ex); } } catch (Exception e) { LOG.error("Error when validating", e); } finally { summaryWorking.stop(); threadRunner.interrupt(); dataPreparationToolGUI.getXmlEadListLabel().repaint(); dataPreparationToolGUI.getXmlEadList().repaint(); if (progressBar != null) { progressBar.setVisible(false); } } if (numberOfFiles == 1) { uniqueFileInstance = fileInstance; } } } } Toolkit.getDefaultToolkit().beep(); if (progressFrame != null) { try { progressFrame.stop(); } catch (Exception e) { LOG.error("Error when stopping the progress bar", e); } } dataPreparationToolGUI.getFinalAct().run(); if (numberOfFiles > 1) { dataPreparationToolGUI.getXmlEadList().clearSelection(); } else if (uniqueFileInstance != null) { if (type != VALIDATE) { dataPreparationToolGUI.getAPEPanel().getApeTabbedPane() .setConversionErrorText(replaceGtAndLt(uniqueFileInstance.getConversionErrors())); if (uniqueXslMessage.equals("")) { dataPreparationToolGUI.getAPEPanel().getApeTabbedPane() .checkFlashingTab(APETabbedPane.TAB_CONVERSION, Utilities.FLASHING_GREEN_COLOR); } else { dataPreparationToolGUI.getAPEPanel().getApeTabbedPane() .checkFlashingTab(APETabbedPane.TAB_CONVERSION, Utilities.FLASHING_RED_COLOR); } } if (type != CONVERT) { dataPreparationToolGUI.getAPEPanel().getApeTabbedPane() .setValidationErrorText(uniqueFileInstance.getValidationErrors()); if (uniqueFileInstance.isValid()) { dataPreparationToolGUI.getAPEPanel().getApeTabbedPane() .checkFlashingTab(APETabbedPane.TAB_VALIDATION, Utilities.FLASHING_GREEN_COLOR); if (uniqueFileInstance.getValidationSchema() .equals(Utilities.getXsdObjectFromPath(Xsd_enum.XSD_APE_SCHEMA.getPath()))) { dataPreparationToolGUI.getAPEPanel().getApeTabbedPane().enableConversionEdmBtn(); dataPreparationToolGUI.getAPEPanel().getApeTabbedPane().enableValidationReportBtn(); } else if (uniqueFileInstance.getValidationSchema() .equals(Utilities.getXsdObjectFromPath(Xsd_enum.XSD_EAD_SCHEMA.getPath())) || uniqueFileInstance.getValidationSchema() .equals(Utilities.getXsdObjectFromPath(Xsd_enum.DTD_EAD_2002.getPath())) || uniqueFileInstance.getValidationSchema().equals( Utilities.getXsdObjectFromPath(Xsd_enum.XSD_EAC_SCHEMA.getPath()))) { dataPreparationToolGUI.getAPEPanel().getApeTabbedPane().enableConversionBtn(); // dataPreparationToolGUI.getAPEPanel().getApeTabbedPane().enableValidationReportBtn(); } } else { dataPreparationToolGUI.getAPEPanel().getApeTabbedPane() .checkFlashingTab(APETabbedPane.TAB_VALIDATION, Utilities.FLASHING_RED_COLOR); if (uniqueFileInstance.getValidationSchema() .equals(Utilities.getXsdObjectFromPath(Xsd_enum.XSD_APE_SCHEMA.getPath())) || uniqueFileInstance.getValidationSchema().equals( Utilities.getXsdObjectFromPath(Xsd_enum.XSD_EAD_SCHEMA.getPath())) || uniqueFileInstance.getValidationSchema() .equals(Utilities.getXsdObjectFromPath(Xsd_enum.DTD_EAD_2002.getPath())) || uniqueFileInstance.getValidationSchema().equals( Utilities.getXsdObjectFromPath(Xsd_enum.XSD_APE_EAC_SCHEMA.getPath())) || uniqueFileInstance.getValidationSchema().equals( Utilities.getXsdObjectFromPath(Xsd_enum.XSD_EAC_SCHEMA.getPath()))) { dataPreparationToolGUI.enableConversionBtns(); dataPreparationToolGUI.getAPEPanel().getApeTabbedPane() .enableConvertAndValidateBtn(); } } } dataPreparationToolGUI.enableMessageReportBtns(); } if (continueLoop) { dataPreparationToolGUI.setResultAreaText(labels.getString("finished")); } else { dataPreparationToolGUI.setResultAreaText(labels.getString("aborted")); } dataPreparationToolGUI.enableSaveBtn(); if (type == CONVERT) { dataPreparationToolGUI.enableValidationBtns(); } dataPreparationToolGUI.enableRadioButtons(); dataPreparationToolGUI.enableEditionTab(); } }).start(); }
From source file:com.aerospike.load.AerospikeLoad.java
public static void main(String[] args) throws IOException { Thread statPrinter = new Thread(new PrintStat(counters)); try {/* w w w .j av a2 s . c o m*/ log.info("Aerospike loader started"); Options options = new Options(); options.addOption("h", "host", true, "Server hostname (default: localhost)"); options.addOption("p", "port", true, "Server port (default: 3000)"); options.addOption("n", "namespace", true, "Namespace (default: test)"); options.addOption("s", "set", true, "Set name. (default: null)"); options.addOption("c", "config", true, "Column definition file name"); options.addOption("wt", "write-threads", true, "Number of writer threads (default: Number of cores * 5)"); options.addOption("rt", "read-threads", true, "Number of reader threads (default: Number of cores * 1)"); options.addOption("l", "rw-throttle", true, "Throttling of reader to writer(default: 10k) "); options.addOption("tt", "transaction-timeout", true, "write transaction timeout in miliseconds(default: No timeout)"); options.addOption("et", "expiration-time", true, "Expiration time of records in seconds (default: never expire)"); options.addOption("T", "timezone", true, "Timezone of source where data dump is taken (default: local timezone)"); options.addOption("ec", "abort-error-count", true, "Error count to abort (default: 0)"); options.addOption("wa", "write-action", true, "Write action if key already exists (default: update)"); options.addOption("v", "verbose", false, "Logging all"); options.addOption("u", "usage", false, "Print usage."); CommandLineParser parser = new PosixParser(); CommandLine cl = parser.parse(options, args, false); if (args.length == 0 || cl.hasOption("u")) { logUsage(options); return; } if (cl.hasOption("l")) { rwThrottle = Integer.parseInt(cl.getOptionValue("l")); } else { rwThrottle = Constants.READLOAD; } // Get all command line options params = Utils.parseParameters(cl); //Get client instance AerospikeClient client = new AerospikeClient(params.host, params.port); if (!client.isConnected()) { log.error("Client is not able to connect:" + params.host + ":" + params.port); return; } if (params.verbose) { log.setLevel(Level.DEBUG); } // Get available processors to calculate default number of threads int cpus = Runtime.getRuntime().availableProcessors(); nWriterThreads = cpus * scaleFactor; nReaderThreads = cpus; // Get writer thread count if (cl.hasOption("wt")) { nWriterThreads = Integer.parseInt(cl.getOptionValue("wt")); nWriterThreads = (nWriterThreads > 0 ? (nWriterThreads > Constants.MAX_THREADS ? Constants.MAX_THREADS : nWriterThreads) : 1); log.debug("Using writer Threads: " + nWriterThreads); } writerPool = Executors.newFixedThreadPool(nWriterThreads); // Get reader thread count if (cl.hasOption("rt")) { nReaderThreads = Integer.parseInt(cl.getOptionValue("rt")); nReaderThreads = (nReaderThreads > 0 ? (nReaderThreads > Constants.MAX_THREADS ? Constants.MAX_THREADS : nReaderThreads) : 1); log.debug("Using reader Threads: " + nReaderThreads); } String columnDefinitionFileName = cl.getOptionValue("c", null); log.debug("Column definition files/directory: " + columnDefinitionFileName); if (columnDefinitionFileName == null) { log.error("Column definition files/directory not specified. use -c <file name>"); return; } File columnDefinitionFile = new File(columnDefinitionFileName); if (!columnDefinitionFile.exists()) { log.error("Column definition files/directory does not exist: " + Utils.getFileName(columnDefinitionFileName)); return; } // Get data file list String[] files = cl.getArgs(); if (files.length == 0) { log.error("No data file Specified: add <file/dir name> to end of the command "); return; } List<String> allFileNames = new ArrayList<String>(); allFileNames = Utils.getFileNames(files); if (allFileNames.size() == 0) { log.error("Given datafiles/directory does not exist"); return; } for (int i = 0; i < allFileNames.size(); i++) { log.debug("File names:" + Utils.getFileName(allFileNames.get(i))); File file = new File(allFileNames.get(i)); counters.write.recordTotal = counters.write.recordTotal + file.length(); } //remove column definition file from list allFileNames.remove(columnDefinitionFileName); log.info("Number of data files:" + allFileNames.size()); /** * Process column definition file to get meta data and bin mapping. */ metadataColumnDefs = new ArrayList<ColumnDefinition>(); binColumnDefs = new ArrayList<ColumnDefinition>(); metadataConfigs = new HashMap<String, String>(); if (Parser.processJSONColumnDefinitions(columnDefinitionFile, metadataConfigs, metadataColumnDefs, binColumnDefs, params)) { log.info("Config file processed."); } else { throw new Exception("Config file parsing Error"); } // Add metadata of config to parameters String metadata; if ((metadata = metadataConfigs.get(Constants.INPUT_TYPE)) != null) { params.fileType = metadata; if (params.fileType.equals(Constants.CSV_FILE)) { // Version check metadata = metadataConfigs.get(Constants.VERSION); String[] vNumber = metadata.split("\\."); int v1 = Integer.parseInt(vNumber[0]); int v2 = Integer.parseInt(vNumber[1]); if ((v1 <= Constants.MajorV) && (v2 <= Constants.MinorV)) { log.debug("Config version used:" + metadata); } else throw new Exception("\"" + Constants.VERSION + ":" + metadata + "\" is not Supported"); // Set delimiter if ((metadata = metadataConfigs.get(Constants.DELIMITER)) != null && metadata.length() == 1) { params.delimiter = metadata.charAt(0); } else { log.warn("\"" + Constants.DELIMITER + ":" + metadata + "\" is not properly specified in config file. Default is ','"); } if ((metadata = metadataConfigs.get(Constants.IGNORE_FIRST_LINE)) != null) { params.ignoreFirstLine = metadata.equals("true"); } else { log.warn("\"" + Constants.IGNORE_FIRST_LINE + ":" + metadata + "\" is not properly specified in config file. Default is false"); } if ((metadata = metadataConfigs.get(Constants.COLUMNS)) != null) { counters.write.colTotal = Integer.parseInt(metadata); } else { throw new Exception("\"" + Constants.COLUMNS + ":" + metadata + "\" is not properly specified in config file"); } } else { throw new Exception("\"" + params.fileType + "\" is not supported in config file"); } } else { throw new Exception("\"" + Constants.INPUT_TYPE + "\" is not specified in config file"); } // add config input to column definitions if (params.fileType.equals(Constants.CSV_FILE)) { List<String> binName = null; if (params.ignoreFirstLine) { String line; BufferedReader br = new BufferedReader( new InputStreamReader(new FileInputStream(allFileNames.get(0)), "UTF8")); if ((line = br.readLine()) != null) { binName = Parser.getCSVRawColumns(line, params.delimiter); br.close(); if (binName.size() != counters.write.colTotal) { throw new Exception("Number of column in config file and datafile are mismatch." + " Datafile: " + Utils.getFileName(allFileNames.get(0)) + " Configfile: " + Utils.getFileName(columnDefinitionFileName)); } } } //update columndefs for metadata for (int i = 0; i < metadataColumnDefs.size(); i++) { if (metadataColumnDefs.get(i).staticValue) { } else { if (metadataColumnDefs.get(i).binValuePos < 0) { if (metadataColumnDefs.get(i).columnName == null) { if (metadataColumnDefs.get(i).jsonPath == null) { log.error("dynamic metadata having improper info" + metadataColumnDefs.toString()); //TODO } else { //TODO check for json_path } } else { if (params.ignoreFirstLine) { if (binName.indexOf(metadataColumnDefs.get(i).binValueHeader) != -1) { metadataColumnDefs.get(i).binValuePos = binName .indexOf(metadataColumnDefs.get(i).binValueHeader); } else { throw new Exception("binName missing in data file:" + metadataColumnDefs.get(i).binValueHeader); } } } } else { if (params.ignoreFirstLine) metadataColumnDefs.get(i).binValueHeader = binName .get(metadataColumnDefs.get(i).binValuePos); } } if ((!metadataColumnDefs.get(i).staticValue) && (metadataColumnDefs.get(i).binValuePos < 0)) { throw new Exception("Information for bin mapping is missing in config file:" + metadataColumnDefs.get(i)); } if (metadataColumnDefs.get(i).srcType == null) { throw new Exception( "Source data type is not properly mentioned:" + metadataColumnDefs.get(i)); } if (metadataColumnDefs.get(i).binNameHeader == Constants.SET && !metadataColumnDefs.get(i).srcType.equals(SrcColumnType.STRING)) { throw new Exception("Set name should be string type:" + metadataColumnDefs.get(i)); } if (metadataColumnDefs.get(i).binNameHeader.equalsIgnoreCase(Constants.SET) && params.set != null) { throw new Exception( "Set name is given both in config file and commandline. Provide only once."); } } //update columndefs for bins for (int i = 0; i < binColumnDefs.size(); i++) { if (binColumnDefs.get(i).staticName) { } else { if (binColumnDefs.get(i).binNamePos < 0) { if (binColumnDefs.get(i).columnName == null) { if (binColumnDefs.get(i).jsonPath == null) { log.error("dynamic bin having improper info"); //TODO } else { //TODO check for json_path } } else { if (params.ignoreFirstLine) { if (binName.indexOf(binColumnDefs.get(i).binNameHeader) != -1) { binColumnDefs.get(i).binNamePos = binName .indexOf(binColumnDefs.get(i).binNameHeader); } else { throw new Exception("binName missing in data file:" + binColumnDefs.get(i).binNameHeader); } } } } else { if (params.ignoreFirstLine) binColumnDefs.get(i).binNameHeader = binName.get(binColumnDefs.get(i).binNamePos); } } if (binColumnDefs.get(i).staticValue) { } else { if (binColumnDefs.get(i).binValuePos < 0) { if (binColumnDefs.get(i).columnName == null) { if (binColumnDefs.get(i).jsonPath == null) { log.error("dynamic bin having improper info"); //TODO } else { //TODO check for json_path } } else { if (params.ignoreFirstLine) { if (binName.contains(binColumnDefs.get(i).binValueHeader)) { binColumnDefs.get(i).binValuePos = binName .indexOf(binColumnDefs.get(i).binValueHeader); } else if (!binColumnDefs.get(i).binValueHeader.toLowerCase() .equals(Constants.SYSTEM_TIME)) { throw new Exception("Wrong column name mentioned in config file:" + binColumnDefs.get(i).binValueHeader); } } } } else { if (params.ignoreFirstLine) binColumnDefs.get(i).binValueHeader = binName.get(binColumnDefs.get(i).binValuePos); } //check for missing entries in config file if (binColumnDefs.get(i).binValuePos < 0 && binColumnDefs.get(i).binValueHeader == null) { throw new Exception("Information missing(Value header or bin mapping) in config file:" + binColumnDefs.get(i)); } //check for proper data type in config file. if (binColumnDefs.get(i).srcType == null) { throw new Exception( "Source data type is not properly mentioned:" + binColumnDefs.get(i)); } //check for valid destination type if ((binColumnDefs.get(i).srcType.equals(SrcColumnType.TIMESTAMP) || binColumnDefs.get(i).srcType.equals(SrcColumnType.BLOB)) && binColumnDefs.get(i).dstType == null) { throw new Exception("Destination type is not mentioned: " + binColumnDefs.get(i)); } //check for encoding if (binColumnDefs.get(i).dstType != null && binColumnDefs.get(i).encoding == null) { throw new Exception( "Encoding is not given for src-dst type conversion:" + binColumnDefs.get(i)); } //check for valid encoding if (binColumnDefs.get(i).srcType.equals(SrcColumnType.BLOB) && !binColumnDefs.get(i).encoding.equals(Constants.HEX_ENCODING)) { throw new Exception("Wrong encoding for blob data:" + binColumnDefs.get(i)); } } //Check static bin name mapped to dynamic bin value if ((binColumnDefs.get(i).binNamePos == binColumnDefs.get(i).binValuePos) && (binColumnDefs.get(i).binNamePos != -1)) { throw new Exception("Static bin name mapped to dynamic bin value:" + binColumnDefs.get(i)); } //check for missing entries in config file if (binColumnDefs.get(i).binNameHeader == null && binColumnDefs.get(i).binNameHeader.length() > Constants.BIN_NAME_LENGTH) { throw new Exception("Information missing binName or large binName in config file:" + binColumnDefs.get(i)); } } } log.info(params.toString()); log.debug("MetadataConfig:" + metadataColumnDefs); log.debug("BinColumnDefs:" + binColumnDefs); // Start PrintStat thread statPrinter.start(); // Reader pool size ExecutorService readerPool = Executors.newFixedThreadPool( nReaderThreads > allFileNames.size() ? allFileNames.size() : nReaderThreads); log.info("Reader pool size : " + nReaderThreads); // Submit all tasks to writer threadpool. for (String aFile : allFileNames) { log.debug("Submitting task for: " + aFile); readerPool.submit(new AerospikeLoad(aFile, client, params)); } // Wait for reader pool to complete readerPool.shutdown(); log.info("Shutdown down reader thread pool"); while (!readerPool.isTerminated()) ; //readerPool.awaitTermination(20, TimeUnit.MINUTES); log.info("Reader thread pool terminated"); // Wait for writer pool to complete after getting all tasks from reader pool writerPool.shutdown(); log.info("Shutdown down writer thread pool"); while (!writerPool.isTerminated()) ; log.info("Writer thread pool terminated"); // Print final statistic of aerospike-loader. log.info("Final Statistics of importer: (Succesfull Writes = " + counters.write.writeCount.get() + ", " + "Errors=" + (counters.write.writeErrors.get() + counters.write.readErrors.get() + counters.write.processingErrors.get()) + "(" + (counters.write.writeErrors.get()) + "-Write," + counters.write.readErrors.get() + "-Read," + counters.write.processingErrors.get() + "-Processing)"); } catch (Exception e) { log.error(e); if (log.isDebugEnabled()) { e.printStackTrace(); } } finally { // Stop statistic printer thread. statPrinter.interrupt(); log.info("Aerospike loader completed"); } }