List of usage examples for java.util Properties putAll
@Override
public synchronized void putAll(Map<?, ?> t)
From source file:org.apache.maven.model.merge.ModelMerger.java
protected void mergeNotifier_Configuration(Notifier target, Notifier source, boolean sourceDominant, Map<Object, Object> context) { Properties merged = new Properties(); if (sourceDominant) { merged.putAll(target.getConfiguration()); merged.putAll(source.getConfiguration()); } else {/* ww w . ja v a 2 s . c o m*/ merged.putAll(source.getConfiguration()); merged.putAll(target.getConfiguration()); } target.setConfiguration(merged); }
From source file:gtu._work.etc.EnglishTester.java
private void initGUI() { try {// w ww. ja v a 2s . c o m JCommonUtil.defaultToolTipDelay(); BorderLayout thisLayout = new BorderLayout(); getContentPane().setLayout(thisLayout); this.setDefaultCloseOperation(WindowConstants.EXIT_ON_CLOSE); { jTabbedPane1 = new JTabbedPane(); getContentPane().add(jTabbedPane1, BorderLayout.CENTER); jTabbedPane1.setPreferredSize(new java.awt.Dimension(462, 259)); jTabbedPane1.addMouseListener(new MouseAdapter() { public void mouseClicked(MouseEvent evt) { // XXX // XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXx jTabbedPane1.requestFocus();// FOCUS TODO // XXX // XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXx } }); jTabbedPane1.addKeyListener(new KeyAdapter() { public void keyPressed(KeyEvent evt) { System.out.println("2===" + evt.getKeyCode()); if (evt.getKeyCode() == 49) {// 0 jTabbedPane1.setSelectedIndex(0); } if (evt.getKeyCode() == 50) {// 1 jTabbedPane1.setSelectedIndex(1); } if (evt.getKeyCode() == 10) {// enter skipBtnAction(); } if (evt.getKeyCode() == 32) {// removeBtnAction(); } } }); { jPanel1 = new JPanel(); BorderLayout jPanel1Layout = new BorderLayout(); jPanel1.setLayout(jPanel1Layout); jTabbedPane1.addTab("english", null, jPanel1, null); { jScrollPane1 = new JScrollPane(); jPanel1.add(jScrollPane1, BorderLayout.CENTER); jScrollPane1.setPreferredSize(new java.awt.Dimension(420, 141)); { englishArea = new JTextArea(); jScrollPane1.setViewportView(englishArea); englishArea.setFont(new java.awt.Font("Microsoft JhengHei", 0, 22)); } } { jPanel5 = new JPanel(); jPanel1.add(jPanel5, BorderLayout.SOUTH); jPanel5.setPreferredSize(new java.awt.Dimension(402, 65)); { skipBtn = new JButton(); jPanel5.add(skipBtn); skipBtn.setText("skip"); skipBtn.setPreferredSize(new java.awt.Dimension(187, 24)); skipBtn.addActionListener(new ActionListener() { public void actionPerformed(ActionEvent evt) { skipBtnAction(); } }); } { removeBtn = new JButton(); jPanel5.add(removeBtn); removeBtn.setText("remove"); removeBtn.setPreferredSize(new java.awt.Dimension(180, 24)); removeBtn.addActionListener(new ActionListener() { public void actionPerformed(ActionEvent evt) { removeBtnAction(); } }); } { questionCountLabel = new JLabel(); jPanel5.add(questionCountLabel); questionCountLabel.setPreferredSize(new java.awt.Dimension(47, 21)); questionCountLabel.setBorder(BorderFactory.createBevelBorder(BevelBorder.LOWERED)); questionCountLabel.setToolTipText(""); } { propCountLabel = new JLabel(); jPanel5.add(propCountLabel); propCountLabel.setPreferredSize(new java.awt.Dimension(45, 21)); propCountLabel.setBorder(BorderFactory.createBevelBorder(BevelBorder.LOWERED)); propCountLabel.setToolTipText(""); } { googleSearchBtn = new JButton(); jPanel5.add(googleSearchBtn); googleSearchBtn.setText("<html>GPic</html>"); googleSearchBtn.setPreferredSize(new java.awt.Dimension(58, 24)); googleSearchBtn.addActionListener(new ActionListener() { public void actionPerformed(ActionEvent evt) { try { String word = currentWordIndex.trim(); ClipboardUtil.getInstance().setContents(word); word = word.replace(" ", "%20"); URI uri = new URI( "https://www.google.com.tw/search?num=10&hl=zh-TW&site=imghp&tbm=isch&source=hp&biw=1280&bih=696&q=" + word); //URI uri = new URI("http://image.baidu.com/search/index?tn=baiduimage&ps=1&ct=201326592&lm=-1&cl=2&nc=1&ie=utf-8&word=" + word); Desktop.getDesktop().browse(uri); } catch (Exception ex) { JCommonUtil.handleException(ex); } } }); } { yahooDicBtn = new JButton(); jPanel5.add(yahooDicBtn); yahooDicBtn.setText("<html>Dict</html>"); yahooDicBtn.setPreferredSize(new java.awt.Dimension(57, 24)); yahooDicBtn.addActionListener(new ActionListener() { public void actionPerformed(ActionEvent evt) { try { // URI uri = new // URI("http://tw.dictionary.yahoo.com/dictionary?p=" // + currentWord.trim()); URI uri = new URI("http://www.dreye.com/axis/ddict.jsp?ver=big5&dod=0102&w=" + currentWordIndex.trim() + "&x=0&y=0"); Desktop.getDesktop().browse(uri); } catch (Exception ex) { JCommonUtil.handleException(ex); } } }); } { pickBtn = new JButton(); jPanel5.add(pickBtn); pickBtn.setText("<html>+Pick</html>"); pickBtn.setPreferredSize(new java.awt.Dimension(60, 24)); pickBtn.addActionListener(new ActionListener() { public void actionPerformed(ActionEvent evt) { try { String key = currentWordIndex; String value = englishProp.getProperty(currentWordIndex); if (StringUtils.isEmpty(value)) { JCommonUtil._jOptionPane_showMessageDialog_error( "add pick failed : no such word => " + key); } else { pickProp.setProperty(key, value); JCommonUtil._jOptionPane_showMessageDialog_info( "key=" + key + "\nvalue=" + value + "\nsize=" + pickProp.size(), "??"); } } catch (Exception ex) { JCommonUtil.handleException(ex); } } }); } { scanPicBtn = new JButton(); scanPicBtn.addActionListener(new ActionListener() { public void actionPerformed(ActionEvent e) { } }); jPanel5.add(scanPicBtn); scanPicBtn.setPreferredSize(new java.awt.Dimension(46, 24)); scanPicBtn.addMouseListener(new MouseAdapter() { public void mouseClicked(MouseEvent evt) { if (picDir == null) { JCommonUtil._jOptionPane_showMessageDialog_error("picDir is null"); return; } if (picSet != null && picSet.size() > 0) { try { Desktop.getDesktop().open(picSet.iterator().next()); } catch (IOException e) { JCommonUtil.handleException(e); } return; } try { String text = currentWordIndex.trim().toLowerCase(); ClipboardUtil.getInstance().setContents(text); text = text.replace(" ", "%20"); URI uri = new URI( "https://www.google.com.tw/search?num=10&hl=zh-TW&site=imghp&tbm=isch&source=hp&biw=1280&bih=696&q=" + text); //URI uri = new URI("http://image.baidu.com/search/index?tn=baiduimage&ps=1&ct=201326592&lm=-1&cl=2&nc=1&ie=utf-8&word=" + text); Desktop.getDesktop().browse(uri); } catch (Exception ex) { JCommonUtil.handleException(ex); } } }); } { showChineseOption = new JCheckBox(); showChineseOption.setSelected(true); jPanel5.add(showChineseOption); } } } { jPanel2 = new JPanel(); BorderLayout jPanel2Layout = new BorderLayout(); jPanel2.setLayout(jPanel2Layout); jTabbedPane1.addTab("chinese", null, jPanel2, null); jPanel2.setPreferredSize(new java.awt.Dimension(420, 211)); { showEnglishText = new JTextField(); jPanel2.add(showEnglishText, BorderLayout.NORTH); showEnglishText.setEditable(false); } { jPanel10 = new JPanel(); jPanel2.add(jPanel10, BorderLayout.CENTER); } { answerBtn[0] = new JButton(); jPanel10.add(answerBtn[0]); answerBtn[0].setPreferredSize(new java.awt.Dimension(190, 110)); answerBtn[0].addActionListener(new ActionListener() { public void actionPerformed(ActionEvent evt) { answerBtnClick(answerBtn[0]); } }); } { answerBtn[1] = new JButton(); jPanel10.add(answerBtn[1]); answerBtn[1].setPreferredSize(new java.awt.Dimension(190, 110)); answerBtn[1].addActionListener(new ActionListener() { public void actionPerformed(ActionEvent evt) { answerBtnClick(answerBtn[1]); } }); } { answerBtn[2] = new JButton(); jPanel10.add(answerBtn[2]); answerBtn[2].setPreferredSize(new java.awt.Dimension(190, 110)); answerBtn[2].addActionListener(new ActionListener() { public void actionPerformed(ActionEvent evt) { answerBtnClick(answerBtn[2]); } }); } { answerBtn[3] = new JButton(); jPanel10.add(answerBtn[3]); answerBtn[3].setPreferredSize(new java.awt.Dimension(190, 110)); answerBtn[3].addActionListener(new ActionListener() { public void actionPerformed(ActionEvent evt) { answerBtnClick(answerBtn[3]); } }); } { for (int ii = 0; ii < 4; ii++) { answerBtn[ii].setFont(new java.awt.Font("", 0, 14)); } } } { jPanel3 = new JPanel(); BorderLayout jPanel3Layout = new BorderLayout(); jPanel3.setLayout(jPanel3Layout); jTabbedPane1.addTab("word", null, jPanel3, null); { jScrollPane3 = new JScrollPane(); jPanel3.add(jScrollPane3, BorderLayout.CENTER); jScrollPane3.setPreferredSize(new java.awt.Dimension(420, 187)); { propTable = new JTable(); jScrollPane3.setViewportView(propTable); JTableUtil.defaultSetting(propTable); propTable.addMouseListener(new MouseAdapter() { public void mouseClicked(MouseEvent evt) { JPopupMenuUtil.newInstance(propTable) .addJMenuItem(JTableUtil.newInstance(propTable).getDefaultJMenuItems()) .applyEvent(evt).show(); } }); } } { saveBtn = new JButton(); jPanel3.add(saveBtn, BorderLayout.SOUTH); saveBtn.setText("save table"); saveBtn.addActionListener(new ActionListener() { public void actionPerformed(ActionEvent evt) { DefaultTableModel model = JTableUtil.newInstance(propTable).getModel(); for (int ii = 0; ii < model.getRowCount(); ii++) { String key = (String) model.getValueAt(ii, 0); String value = (String) model.getValueAt(ii, 1); if (!englishProp.containsKey(key)) { englishProp.setProperty(key, value); } } try { englishProp.store(new FileOutputStream(englishFile), "comment"); } catch (Exception e) { e.printStackTrace(); } JCommonUtil._jOptionPane_showMessageDialog_info("save file ok! \n" + englishFile); } }); } { queryText = new JTextField(); jPanel3.add(queryText, BorderLayout.NORTH); queryText.getDocument() .addDocumentListener(JCommonUtil.getDocumentListener(new HandleDocumentEvent() { @Override public void process(DocumentEvent event) { String text = JCommonUtil.getDocumentText(event); Pattern pattern = Pattern.compile(text); Matcher matcher = null; DefaultTableModel propTableModel = JTableUtil.createModel(false, "english", "chinese"); for (Enumeration<?> enu = englishProp.propertyNames(); enu .hasMoreElements();) { String key = (String) enu.nextElement(); String value = englishProp.getProperty(key); if (key.contains(text)) { propTableModel.addRow(new Object[] { key, value }); continue; } matcher = pattern.matcher(key); if (matcher.find()) { propTableModel.addRow(new Object[] { key, value }); continue; } } propTable.setModel(propTableModel); } })); } } { jPanel4 = new JPanel(); jTabbedPane1.addTab("config", null, jPanel4, null); { savePickBtn = new JButton(); jPanel4.add(savePickBtn); savePickBtn.setText("save pick"); savePickBtn.setPreferredSize(new java.awt.Dimension(116, 40)); savePickBtn.addActionListener(new ActionListener() { public void actionPerformed(ActionEvent evt) { if (englishFile == null) { File file = new File(// PropertiesUtil.getJarCurrentPath(EnglishTester.class), "temp.properties"); englishFile = file; } if (pickProp.isEmpty()) { JCommonUtil._jOptionPane_showMessageDialog_error("?!"); return; } String fileName = englishFile.getName().replaceAll("\\.properties", "_bak.properties"); File jarWhereFile = PropertiesUtil.getJarCurrentPath(EnglishTester.class); fileName = JCommonUtil._jOptionPane_showInputDialog("save target properties", fileName); if (StringUtils.isEmpty(fileName)) { JCommonUtil._jOptionPane_showMessageDialog_error("can't save!"); return; } if (fileName.equalsIgnoreCase(englishFile.getName())) { JCommonUtil._jOptionPane_showMessageDialog_error( "??englishFile???"); return; } if (!fileName.endsWith(".properties")) { fileName += ".properties"; } File newFile = new File(jarWhereFile, fileName); Properties oldProp = new Properties(); if (newFile.exists()) { try { oldProp.load(new FileInputStream(newFile)); } catch (Exception e) { e.printStackTrace(); } } oldProp.putAll(pickProp); try { oldProp.store(new FileOutputStream(newFile), "comment"); } catch (Exception e) { e.printStackTrace(); } JCommonUtil._jOptionPane_showMessageDialog_info("save file ok! \n" + newFile); } }); } { saveConfigBtn2 = new JButton(); jPanel4.add(saveConfigBtn2); saveConfigBtn2.setText("save config"); saveConfigBtn2.setPreferredSize(new java.awt.Dimension(108, 40)); saveConfigBtn2.addActionListener(new ActionListener() { public void actionPerformed(ActionEvent evt) { saveConfigBtnAction(); } }); } { startAllBtn = new JButton(); jPanel4.add(startAllBtn); startAllBtn.setText("start all"); startAllBtn.setPreferredSize(new java.awt.Dimension(101, 40)); startAllBtn.addActionListener(new ActionListener() { public void actionPerformed(ActionEvent evt) { Object[] files = fileList.getSelectedValues(); if (files == null || files.length == 0) { JCommonUtil ._jOptionPane_showMessageDialog_error("?properties"); return; } Properties allProp = new Properties(); Properties prop = new Properties(); for (Object ff : files) { try { prop.load(new FileInputStream((File) ff)); } catch (Exception e) { JCommonUtil.handleException(e); } allProp.putAll(prop); } englishProp = allProp; System.out.println("englishProp = " + englishProp.size()); startNow(); } }); } { startNow = new JButton(); jPanel4.add(startNow); startNow.setText("start now"); startNow.setPreferredSize(new java.awt.Dimension(101, 40)); startNow.addActionListener(new ActionListener() { public void actionPerformed(ActionEvent evt) { startNow(); } }); } { picOnly = new JCheckBox(); jPanel4.add(picOnly); picOnly.setText("picOnly"); } { sortChkBox = new JCheckBox(); jPanel4.add(sortChkBox); sortChkBox.setText("sort"); } { showPicChkBox = new JCheckBox(); showPicChkBox.setSelected(true); jPanel4.add(showPicChkBox); showPicChkBox.setText("showPic"); } { JCommonUtil.defaultToolTipDelay(); fontSizeSliber = new JSlider(JSlider.HORIZONTAL); jPanel4.add(fontSizeSliber); fontSizeSliber.setPreferredSize(new java.awt.Dimension(419, 35)); fontSizeSliber.setValue(22); fontSizeSliber.setMinimum(22); fontSizeSliber.setMaximum(300); fontSizeSliber.setMajorTickSpacing(30); fontSizeSliber.setMinorTickSpacing(5); fontSizeSliber.setCursor(new Cursor(Cursor.HAND_CURSOR)); fontSizeSliber.setPaintTicks(false); fontSizeSliber.setPaintLabels(true); { picFolderDirText = new JTextField(); JCommonUtil.jTextFieldSetFilePathMouseEvent(picFolderDirText, true); jPanel4.add(picFolderDirText); picFolderDirText.setColumns(20); } fontSizeSliber.addChangeListener(new ChangeListener() { @Override public void stateChanged(ChangeEvent e) { int size = fontSizeSliber.getValue(); fontSizeSliber.setToolTipText("" + size); englishArea.setFont(new java.awt.Font("Microsoft JhengHei", 0, size)); } }); } } { jPanel6 = new JPanel(); jTabbedPane1.addTab("files", null, jPanel6, null); BorderLayout jPanel6Layout = new BorderLayout(); jPanel6.setLayout(jPanel6Layout); { jScrollPane4 = new JScrollPane(); jPanel6.add(jScrollPane4, BorderLayout.CENTER); jScrollPane4.setPreferredSize(new java.awt.Dimension(420, 211)); { fileList = new JList(); reloadFileList(); jScrollPane4.setViewportView(fileList); fileList.addMouseListener(new MouseAdapter() { public void mouseClicked(MouseEvent evt) { final File file = JListUtil.getLeadSelectionObject(fileList); if (JMouseEventUtil.buttonRightClick(1, evt)) { JPopupMenuUtil.newInstance(EnglishTester.this.fileList).applyEvent(evt)// .addJMenuItem("reload", new ActionListener() { @Override public void actionPerformed(ActionEvent e) { reloadFileList(); } })// .addJMenuItem("delete : " + file.getName(), new ActionListener() { @Override public void actionPerformed(ActionEvent e) { boolean result = JCommonUtil ._JOptionPane_showConfirmDialog_yesNoOption( "delete : " + file.getName() + " ?", "confirm"); if (result) { file.delete(); reloadFileList(); } }// }).show(); return; } if (evt.getClickCount() == 1) { return; } if (JCommonUtil._JOptionPane_showConfirmDialog_yesNoOption( "?,?\n" + file.getName(), "")) { loadEnglishFile(file); } } }); fileList.addKeyListener(new KeyAdapter() { public void keyPressed(KeyEvent evt) { JListUtil.newInstance(fileList).defaultJListKeyPressed(evt); } }); } } } { jPanel9 = new JPanel(); jTabbedPane1.addTab("pic", null, jPanel9, null); { picCheckText = new JTextField(); jPanel9.add(picCheckText); picCheckText.setPreferredSize(new java.awt.Dimension(177, 39)); } { picCheckBtn = new JButton(); jPanel9.add(picCheckBtn); picCheckBtn.setText("check"); picCheckBtn.setPreferredSize(new java.awt.Dimension(98, 43)); { jPanel11 = new JPanel(); jTabbedPane1.addTab("", null, jPanel11, null); jPanel11.setLayout(new BorderLayout(0, 0)); { inputTestArea2 = new JTextArea(); inputTestArea2.setFont(new Font("", Font.PLAIN, 12)); inputTestArea2.addKeyListener(new KeyAdapter() { @Override public void keyReleased(KeyEvent e) { inputTestTrainer.keyin(e); } }); jPanel11.add(inputTestArea2, BorderLayout.SOUTH); } { inputTestArea1 = new JTextArea(); JTextAreaUtil.setWrapTextArea(inputTestArea1); inputTestArea1.setFont(new Font("", Font.PLAIN, 22)); jPanel11.add(inputTestArea1, BorderLayout.CENTER); } { panel = new JPanel(); jPanel11.add(panel, BorderLayout.NORTH); { inputTestLabel = new JLabel(""); panel.add(inputTestLabel); } { inputTestChk = new JCheckBox(""); inputTestChk.addActionListener(new ActionListener() { public void actionPerformed(ActionEvent e) { inputTestTrainer.initQuestion(); } }); panel.add(inputTestChk); } } } picCheckBtn.addActionListener(new ActionListener() { void scanPic(String searchWord, File file, Set<File> findFile) { if (file.isDirectory()) { File[] list = null; if ((list = file.listFiles()) != null) { for (File f : list) { scanPic(searchWord, f, findFile); } } } else { String text = searchWord; String name = file.getName().toLowerCase(); if (isMatch(name, text)) { findFile.add(file); } } } public void actionPerformed(ActionEvent evt) { picDir = new File(picFolderDirText.getText()); if (picDir == null) { JCommonUtil._jOptionPane_showMessageDialog_error("picDir is null"); return; } if (!picDir.exists() || !picDir.isDirectory()) { JCommonUtil._jOptionPane_showMessageDialog_error("picDir "); return; } picCheckBtn.setText("search.."); String searchWord = picCheckText.getText().toLowerCase().trim(); Set<File> picSet2 = new HashSet<File>(); scanPic(searchWord, picDir, picSet2); if (picSet2 != null && picSet2.size() > 0) { picCheckBtn.setText("" + picSet2.size()); try { Desktop.getDesktop().open(picSet2.iterator().next()); } catch (IOException e) { JCommonUtil.handleException(e); } } else { picCheckBtn.setText("0"); } } }); } } } JCommonUtil.setJFrameIcon(this, "resource/images/ico/english_tester.ico"); pack(); this.setSize(423, 314); configHelper.init(); } catch (Exception e) { // add your error handling code here e.printStackTrace(); } }
From source file:org.apache.maven.model.merge.ModelMerger.java
protected void mergeModelBase_Properties(ModelBase target, ModelBase source, boolean sourceDominant, Map<Object, Object> context) { Properties merged = new Properties(); if (sourceDominant) { merged.putAll(target.getProperties()); merged.putAll(source.getProperties()); } else {//from www.j a v a 2 s. c o m merged.putAll(source.getProperties()); merged.putAll(target.getProperties()); } target.setProperties(merged); target.setLocation("properties", InputLocation.merge(target.getLocation("properties"), source.getLocation("properties"), sourceDominant)); }
From source file:org.apache.maven.model.merge.ModelMerger.java
protected void mergeContributor_Properties(Contributor target, Contributor source, boolean sourceDominant, Map<Object, Object> context) { Properties merged = new Properties(); if (sourceDominant) { merged.putAll(target.getProperties()); merged.putAll(source.getProperties()); } else {/*from ww w .java 2s . com*/ merged.putAll(source.getProperties()); merged.putAll(target.getProperties()); } target.setProperties(merged); target.setLocation("properties", InputLocation.merge(target.getLocation("properties"), source.getLocation("properties"), sourceDominant)); }
From source file:org.codehaus.groovy.grails.orm.hibernate.cfg.GrailsDomainBinder.java
@SuppressWarnings("unchecked") private static void bindSimpleId(GrailsDomainClassProperty identifier, RootClass entity, Mappings mappings, Identity mappedId, String sessionFactoryBeanName) { // create the id value SimpleValue id = new SimpleValue(mappings, entity.getTable()); // set identifier on entity Properties params = new Properties(); entity.setIdentifier(id);/*from ww w. ja va 2s .c om*/ if (mappedId == null) { // configure generator strategy id.setIdentifierGeneratorStrategy("native"); } else { id.setIdentifierGeneratorStrategy(mappedId.getGenerator()); params.putAll(mappedId.getParams()); if ("assigned".equals(mappedId.getGenerator())) { id.setNullValue("undefined"); } } params.put(PersistentIdentifierGenerator.IDENTIFIER_NORMALIZER, mappings.getObjectNameNormalizer()); if (mappings.getSchemaName() != null) { params.setProperty(PersistentIdentifierGenerator.SCHEMA, mappings.getSchemaName()); } if (mappings.getCatalogName() != null) { params.setProperty(PersistentIdentifierGenerator.CATALOG, mappings.getCatalogName()); } id.setIdentifierGeneratorProperties(params); // bind value bindSimpleValue(identifier, null, id, EMPTY_PATH, mappings, sessionFactoryBeanName); // create property Property prop = new Property(); prop.setValue(id); // bind property bindProperty(identifier, prop, mappings); // set identifier property entity.setIdentifierProperty(prop); id.getTable().setIdentifierValue(id); }
From source file:org.apache.flink.streaming.connectors.kafka.KafkaConsumerTestBase.java
/** * Test delete behavior and metrics for producer * @throws Exception/*from w w w .j a va 2 s .c o m*/ */ public void runAllDeletesTest() throws Exception { final String topic = "alldeletestest"; createTestTopic(topic, 1, 1); final int ELEMENT_COUNT = 300; // ----------- Write some data into Kafka ------------------- StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment("localhost", flinkPort); env.setParallelism(1); env.getConfig().setRestartStrategy(RestartStrategies.noRestart()); env.getConfig().disableSysoutLogging(); DataStream<Tuple2<byte[], PojoValue>> kvStream = env .addSource(new SourceFunction<Tuple2<byte[], PojoValue>>() { @Override public void run(SourceContext<Tuple2<byte[], PojoValue>> ctx) throws Exception { Random rnd = new Random(1337); for (long i = 0; i < ELEMENT_COUNT; i++) { final byte[] key = new byte[200]; rnd.nextBytes(key); ctx.collect(new Tuple2<>(key, (PojoValue) null)); } } @Override public void cancel() { } }); TypeInformationKeyValueSerializationSchema<byte[], PojoValue> schema = new TypeInformationKeyValueSerializationSchema<>( byte[].class, PojoValue.class, env.getConfig()); Properties producerProperties = FlinkKafkaProducerBase.getPropertiesFromBrokerList(brokerConnectionStrings); producerProperties.setProperty("retries", "3"); producerProperties.putAll(secureProps); kafkaServer.produceIntoKafka(kvStream, topic, schema, producerProperties, null); env.execute("Write deletes to Kafka"); // ----------- Read the data again ------------------- env = StreamExecutionEnvironment.createRemoteEnvironment("localhost", flinkPort); env.setParallelism(1); env.getConfig().setRestartStrategy(RestartStrategies.noRestart()); env.getConfig().disableSysoutLogging(); Properties props = new Properties(); props.putAll(standardProps); props.putAll(secureProps); DataStream<Tuple2<byte[], PojoValue>> fromKafka = env .addSource(kafkaServer.getConsumer(topic, schema, props)); fromKafka.flatMap(new RichFlatMapFunction<Tuple2<byte[], PojoValue>, Object>() { long counter = 0; @Override public void flatMap(Tuple2<byte[], PojoValue> value, Collector<Object> out) throws Exception { // ensure that deleted messages are passed as nulls assertNull(value.f1); counter++; if (counter == ELEMENT_COUNT) { // we got the right number of elements throw new SuccessException(); } } }); tryExecute(env, "Read deletes from Kafka"); deleteTestTopic(topic); }
From source file:org.apache.flink.streaming.connectors.kafka.KafkaConsumerTestBase.java
/** * Ensure Kafka is working on both producer and consumer side. * This executes a job that contains two Flink pipelines. * * <pre>//w w w .ja va 2s. co m * (generator source) --> (kafka sink)-[KAFKA-TOPIC]-(kafka source) --> (validating sink) * </pre> * * We need to externally retry this test. We cannot let Flink's retry mechanism do it, because the Kafka producer * does not guarantee exactly-once output. Hence a recovery would introduce duplicates that * cause the test to fail. * * This test also ensures that FLINK-3156 doesn't happen again: * * The following situation caused a NPE in the FlinkKafkaConsumer * * topic-1 <-- elements are only produced into topic1. * topic-2 * * Therefore, this test is consuming as well from an empty topic. * */ @RetryOnException(times = 2, exception = kafka.common.NotLeaderForPartitionException.class) public void runSimpleConcurrentProducerConsumerTopology() throws Exception { final String topic = "concurrentProducerConsumerTopic_" + UUID.randomUUID().toString(); final String additionalEmptyTopic = "additionalEmptyTopic_" + UUID.randomUUID().toString(); final int parallelism = 3; final int elementsPerPartition = 100; final int totalElements = parallelism * elementsPerPartition; createTestTopic(topic, parallelism, 2); createTestTopic(additionalEmptyTopic, parallelism, 1); // create an empty topic which will remain empty all the time final StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment("localhost", flinkPort); env.setParallelism(parallelism); env.enableCheckpointing(500); env.setRestartStrategy(RestartStrategies.noRestart()); // fail immediately env.getConfig().disableSysoutLogging(); TypeInformation<Tuple2<Long, String>> longStringType = TypeInfoParser.parse("Tuple2<Long, String>"); TypeInformationSerializationSchema<Tuple2<Long, String>> sourceSchema = new TypeInformationSerializationSchema<>( longStringType, env.getConfig()); TypeInformationSerializationSchema<Tuple2<Long, String>> sinkSchema = new TypeInformationSerializationSchema<>( longStringType, env.getConfig()); // ----------- add producer dataflow ---------- DataStream<Tuple2<Long, String>> stream = env .addSource(new RichParallelSourceFunction<Tuple2<Long, String>>() { private boolean running = true; @Override public void run(SourceContext<Tuple2<Long, String>> ctx) throws InterruptedException { int cnt = getRuntimeContext().getIndexOfThisSubtask() * elementsPerPartition; int limit = cnt + elementsPerPartition; while (running && cnt < limit) { ctx.collect(new Tuple2<>(1000L + cnt, "kafka-" + cnt)); cnt++; // we delay data generation a bit so that we are sure that some checkpoints are // triggered (for FLINK-3156) Thread.sleep(50); } } @Override public void cancel() { running = false; } }); Properties producerProperties = FlinkKafkaProducerBase.getPropertiesFromBrokerList(brokerConnectionStrings); producerProperties.setProperty("retries", "3"); producerProperties.putAll(secureProps); kafkaServer.produceIntoKafka(stream, topic, new KeyedSerializationSchemaWrapper<>(sinkSchema), producerProperties, null); // ----------- add consumer dataflow ---------- List<String> topics = new ArrayList<>(); topics.add(topic); topics.add(additionalEmptyTopic); Properties props = new Properties(); props.putAll(standardProps); props.putAll(secureProps); FlinkKafkaConsumerBase<Tuple2<Long, String>> source = kafkaServer.getConsumer(topics, sourceSchema, props); DataStreamSource<Tuple2<Long, String>> consuming = env.addSource(source).setParallelism(parallelism); consuming.addSink(new RichSinkFunction<Tuple2<Long, String>>() { private int elCnt = 0; private BitSet validator = new BitSet(totalElements); @Override public void invoke(Tuple2<Long, String> value) throws Exception { String[] sp = value.f1.split("-"); int v = Integer.parseInt(sp[1]); assertEquals(value.f0 - 1000, (long) v); assertFalse("Received tuple twice", validator.get(v)); validator.set(v); elCnt++; if (elCnt == totalElements) { // check if everything in the bitset is set to true int nc; if ((nc = validator.nextClearBit(0)) != totalElements) { fail("The bitset was not set to 1 on all elements. Next clear:" + nc + " Set: " + validator); } throw new SuccessException(); } } @Override public void close() throws Exception { super.close(); } }).setParallelism(1); try { tryExecutePropagateExceptions(env, "runSimpleConcurrentProducerConsumerTopology"); } catch (ProgramInvocationException | JobExecutionException e) { // look for NotLeaderForPartitionException Throwable cause = e.getCause(); // search for nested SuccessExceptions int depth = 0; while (cause != null && depth++ < 20) { if (cause instanceof kafka.common.NotLeaderForPartitionException) { throw (Exception) cause; } cause = cause.getCause(); } throw e; } deleteTestTopic(topic); }
From source file:OneParameterization.java
@Override public void run(CommandLine commandLine) throws IOException { File outputFile = new File(commandLine.getOptionValue("output")); File inputFile = new File(commandLine.getOptionValue("input")); ParameterFile parameterFile = new ParameterFile(new File(commandLine.getOptionValue("parameterFile"))); // sanity check to ensure input hasn't been modified after the output if (!commandLine.hasOption("force") && (outputFile.lastModified() > 0L) && (inputFile.lastModified() > outputFile.lastModified())) { throw new FrameworkException("input appears to be newer than output"); }/* w ww .j a va2s .co m*/ // open the resources and begin processing try { problem = ProblemFactory.getInstance().getProblem(commandLine.getOptionValue("problem")); try { input = new SampleReader(new FileReader(inputFile), parameterFile); try { if (commandLine.hasOption("metrics")) { NondominatedPopulation referenceSet = null; // load reference set and create the quality indicator if (commandLine.hasOption("reference")) { referenceSet = new NondominatedPopulation( PopulationIO.readObjectives(new File(commandLine.getOptionValue("reference")))); } else { referenceSet = ProblemFactory.getInstance() .getReferenceSet(commandLine.getOptionValue("problem")); } if (referenceSet == null) { throw new FrameworkException("no reference set available"); } QualityIndicator indicator = new QualityIndicator(problem, referenceSet); output = new MetricFileWriter(indicator, outputFile); } else { output = new ResultFileWriter(problem, outputFile, !commandLine.hasOption("novariables")); } // resume at the last good output // for (int i = 0; i < output.getNumberOfEntries(); i++) { // if (input.hasNext()) { // input.next(); // } else { // throw new FrameworkException( // "output has more entries than input"); // } // } // only one input. scan forward to it Properties properties = new Properties(); if (commandLine.hasOption("inputnumber")) { long limit = Long.parseLong(commandLine.getOptionValue("inputnumber")); for (long ii = 0; ii <= limit; ii++) { if (input.hasNext()) { properties = input.next(); } else { throw new FrameworkException("input line does not exist"); } } } // setup any default parameters Properties defaultProperties = new Properties(); if (commandLine.hasOption("properties")) { for (String property : commandLine.getOptionValues("properties")) { String[] tokens = property.split("="); if (tokens.length == 2) { defaultProperties.setProperty(tokens[0], tokens[1]); } else { throw new FrameworkException("malformed property argument"); } } } if (commandLine.hasOption("epsilon")) { defaultProperties.setProperty("epsilon", commandLine.getOptionValue("epsilon")); } // seed the pseudo-random number generator if (commandLine.hasOption("seed")) { PRNG.setSeed(Long.parseLong(commandLine.getOptionValue("seed"))); } // process the remaining runs // while (input.hasNext()) { // Properties properties = input.next(); // properties.putAll(defaultProperties); // // process(commandLine.getOptionValue("algorithm"), // properties); // } properties.putAll(defaultProperties); process(commandLine.getOptionValue("algorithm"), properties); } finally { if (output != null) { output.close(); } } } finally { if (input != null) { input.close(); } } } finally { if (problem != null) { problem.close(); } } }
From source file:org.apache.flink.streaming.connectors.kafka.KafkaConsumerTestBase.java
protected String writeSequence(String baseTopicName, final int numElements, final int parallelism, final int replicationFactor) throws Exception { LOG.info("\n===================================\n" + "== Writing sequence of " + numElements + " into " + baseTopicName + " with p=" + parallelism + "\n" + "==================================="); final TypeInformation<Tuple2<Integer, Integer>> resultType = TypeInformation .of(new TypeHint<Tuple2<Integer, Integer>>() { });// w ww . ja va2s. co m final KeyedSerializationSchema<Tuple2<Integer, Integer>> serSchema = new KeyedSerializationSchemaWrapper<>( new TypeInformationSerializationSchema<>(resultType, new ExecutionConfig())); final KeyedDeserializationSchema<Tuple2<Integer, Integer>> deserSchema = new KeyedDeserializationSchemaWrapper<>( new TypeInformationSerializationSchema<>(resultType, new ExecutionConfig())); final int maxNumAttempts = 10; for (int attempt = 1; attempt <= maxNumAttempts; attempt++) { final String topicName = baseTopicName + '-' + attempt; LOG.info("Writing attempt #1"); // -------- Write the Sequence -------- createTestTopic(topicName, parallelism, replicationFactor); StreamExecutionEnvironment writeEnv = StreamExecutionEnvironment.createRemoteEnvironment("localhost", flinkPort); writeEnv.getConfig().setRestartStrategy(RestartStrategies.noRestart()); writeEnv.getConfig().disableSysoutLogging(); DataStream<Tuple2<Integer, Integer>> stream = writeEnv .addSource(new RichParallelSourceFunction<Tuple2<Integer, Integer>>() { private boolean running = true; @Override public void run(SourceContext<Tuple2<Integer, Integer>> ctx) throws Exception { int cnt = 0; int partition = getRuntimeContext().getIndexOfThisSubtask(); while (running && cnt < numElements) { ctx.collect(new Tuple2<>(partition, cnt)); cnt++; } } @Override public void cancel() { running = false; } }).setParallelism(parallelism); // the producer must not produce duplicates Properties producerProperties = FlinkKafkaProducerBase .getPropertiesFromBrokerList(brokerConnectionStrings); producerProperties.setProperty("retries", "0"); producerProperties.putAll(secureProps); kafkaServer.produceIntoKafka(stream, topicName, serSchema, producerProperties, new Tuple2Partitioner(parallelism)).setParallelism(parallelism); try { writeEnv.execute("Write sequence"); } catch (Exception e) { LOG.error("Write attempt failed, trying again", e); deleteTestTopic(topicName); JobManagerCommunicationUtils.waitUntilNoJobIsRunning(flink.getLeaderGateway(timeout)); continue; } LOG.info("Finished writing sequence"); // -------- Validate the Sequence -------- // we need to validate the sequence, because kafka's producers are not exactly once LOG.info("Validating sequence"); JobManagerCommunicationUtils.waitUntilNoJobIsRunning(flink.getLeaderGateway(timeout)); final StreamExecutionEnvironment readEnv = StreamExecutionEnvironment .createRemoteEnvironment("localhost", flinkPort); readEnv.getConfig().setRestartStrategy(RestartStrategies.noRestart()); readEnv.getConfig().disableSysoutLogging(); readEnv.setParallelism(parallelism); Properties readProps = (Properties) standardProps.clone(); readProps.setProperty("group.id", "flink-tests-validator"); readProps.putAll(secureProps); FlinkKafkaConsumerBase<Tuple2<Integer, Integer>> consumer = kafkaServer.getConsumer(topicName, deserSchema, readProps); readEnv.addSource(consumer) .map(new RichMapFunction<Tuple2<Integer, Integer>, Tuple2<Integer, Integer>>() { private final int totalCount = parallelism * numElements; private int count = 0; @Override public Tuple2<Integer, Integer> map(Tuple2<Integer, Integer> value) throws Exception { if (++count == totalCount) { throw new SuccessException(); } else { return value; } } }).setParallelism(1).addSink(new DiscardingSink<Tuple2<Integer, Integer>>()).setParallelism(1); final AtomicReference<Throwable> errorRef = new AtomicReference<>(); Thread runner = new Thread() { @Override public void run() { try { tryExecute(readEnv, "sequence validation"); } catch (Throwable t) { errorRef.set(t); } } }; runner.start(); final long deadline = System.nanoTime() + 10_000_000_000L; long delay; while (runner.isAlive() && (delay = deadline - System.nanoTime()) > 0) { runner.join(delay / 1_000_000L); } boolean success; if (runner.isAlive()) { // did not finish in time, maybe the producer dropped one or more records and // the validation did not reach the exit point success = false; JobManagerCommunicationUtils.cancelCurrentJob(flink.getLeaderGateway(timeout)); } else { Throwable error = errorRef.get(); if (error != null) { success = false; LOG.info("Attempt " + attempt + " failed with exception", error); } else { success = true; } } JobManagerCommunicationUtils.waitUntilNoJobIsRunning(flink.getLeaderGateway(timeout)); if (success) { // everything is good! return topicName; } else { deleteTestTopic(topicName); // fall through the loop } } throw new Exception("Could not write a valid sequence to Kafka after " + maxNumAttempts + " attempts"); }