List of usage examples for java.io DataOutputStream DataOutputStream
public DataOutputStream(OutputStream out)
From source file:com.adobe.aem.demo.Analytics.java
public static void main(String[] args) { String hostname = null;//from w w w .j a va 2 s. c om String url = null; String eventfile = null; // Command line options for this tool Options options = new Options(); options.addOption("h", true, "Hostname"); options.addOption("u", true, "Url"); options.addOption("f", true, "Event data file"); CommandLineParser parser = new BasicParser(); try { CommandLine cmd = parser.parse(options, args); if (cmd.hasOption("u")) { url = cmd.getOptionValue("u"); } if (cmd.hasOption("f")) { eventfile = cmd.getOptionValue("f"); } if (cmd.hasOption("h")) { hostname = cmd.getOptionValue("h"); } if (eventfile == null || hostname == null || url == null) { System.out.println("Command line parameters: -h hostname -u url -f path_to_XML_file"); System.exit(-1); } } catch (ParseException ex) { logger.error(ex.getMessage()); } URLConnection urlConn = null; DataOutputStream printout = null; BufferedReader input = null; String u = "http://" + hostname + "/" + url; String tmp = null; try { URL myurl = new URL(u); urlConn = myurl.openConnection(); urlConn.setDoInput(true); urlConn.setDoOutput(true); urlConn.setUseCaches(false); urlConn.setRequestProperty("Content-Type", "application/x-www-form-urlencoded"); printout = new DataOutputStream(urlConn.getOutputStream()); String xml = readFile(eventfile, StandardCharsets.UTF_8); printout.writeBytes(xml); printout.flush(); printout.close(); input = new BufferedReader(new InputStreamReader(urlConn.getInputStream())); logger.debug(xml); while (null != ((tmp = input.readLine()))) { logger.debug(tmp); } printout.close(); input.close(); } catch (Exception ex) { logger.error(ex.getMessage()); } }
From source file:com.ebay.erl.mobius.core.model.Tuple.java
public static void main(String[] arg) throws Throwable { DataOutputStream dos = new DataOutputStream(new FileOutputStream(new File("s:/test.binary"))); //dos.writeUTF("00_DW_LSTG_ITEM"); dos.writeInt(5);//w w w . j a v a2s.com dos.flush(); dos.close(); }
From source file:com.gomoob.embedded.EmbeddedMongo.java
/** * Main entry of the program./*from w w w .ja v a 2s . c o m*/ * * @param args arguments used to customize starting. * * @throws Exception If an error occured while executing the server. */ public static void main(String[] args) throws Exception { // Creates a default execution context, then the configuration of this context can be changed depending on the // command line parameters received. context = new Context(); // Parse the command line parseCommandLine(args); boolean terminated = false; System.out.println("MONGOD_HOST=" + context.getMongoContext().getNet().getServerAddress().getHostName()); System.out.println("MONGOD_PORT=" + context.getMongoContext().getNet().getPort()); System.out.println("SERVER_SOCKET_PORT=" + context.getSocketContext().getServerSocket().getLocalPort()); Socket socket = null; BufferedReader reader = null; Writer writer = null; while (!terminated) { socket = context.getSocketContext().getServerSocket().accept(); reader = new BufferedReader(new InputStreamReader(socket.getInputStream())); writer = new OutputStreamWriter(new DataOutputStream(socket.getOutputStream())); System.out.println("Waiting for command..."); String commandString = reader.readLine(); System.out.println(commandString); ICommand command = parseCommandString(commandString); IResponse response = command.run(context); writer.write(response.toJSON()); writer.close(); // If the current socket is opened close it if (!socket.isClosed()) { socket.close(); } // If stop is required terminated = response.isTerminationRequired(); } context.getSocketContext().getServerSocket().close(); }
From source file:hyperloglog.tools.HyperLogLogCLI.java
public static void main(String[] args) { Options options = new Options(); addOptions(options);/*from w w w . j av a2 s . c o m*/ CommandLineParser parser = new BasicParser(); CommandLine cli = null; long n = 0; long seed = 123; EncodingType enc = EncodingType.SPARSE; int p = 14; int hb = 64; boolean bitPack = true; boolean noBias = true; int unique = -1; String filePath = null; BufferedReader br = null; String outFile = null; String inFile = null; FileOutputStream fos = null; DataOutputStream out = null; FileInputStream fis = null; DataInputStream in = null; try { cli = parser.parse(options, args); if (!(cli.hasOption('n') || cli.hasOption('f') || cli.hasOption('d'))) { System.out.println("Example usage: hll -n 1000 " + "<OR> hll -f /tmp/input.txt " + "<OR> hll -d -i /tmp/out.hll"); usage(options); return; } if (cli.hasOption('n')) { n = Long.parseLong(cli.getOptionValue('n')); } if (cli.hasOption('e')) { String value = cli.getOptionValue('e'); if (value.equals(EncodingType.DENSE.name())) { enc = EncodingType.DENSE; } } if (cli.hasOption('p')) { p = Integer.parseInt(cli.getOptionValue('p')); if (p < 4 && p > 16) { System.out.println("Warning! Out-of-range value specified for p. Using to p=14."); p = 14; } } if (cli.hasOption('h')) { hb = Integer.parseInt(cli.getOptionValue('h')); } if (cli.hasOption('c')) { noBias = Boolean.parseBoolean(cli.getOptionValue('c')); } if (cli.hasOption('b')) { bitPack = Boolean.parseBoolean(cli.getOptionValue('b')); } if (cli.hasOption('f')) { filePath = cli.getOptionValue('f'); br = new BufferedReader(new FileReader(new File(filePath))); } if (filePath != null && cli.hasOption('n')) { System.out.println("'-f' (input file) specified. Ignoring -n."); } if (cli.hasOption('s')) { if (cli.hasOption('o')) { outFile = cli.getOptionValue('o'); fos = new FileOutputStream(new File(outFile)); out = new DataOutputStream(fos); } else { System.err.println("Specify output file. Example usage: hll -s -o /tmp/out.hll"); usage(options); return; } } if (cli.hasOption('d')) { if (cli.hasOption('i')) { inFile = cli.getOptionValue('i'); fis = new FileInputStream(new File(inFile)); in = new DataInputStream(fis); } else { System.err.println("Specify input file. Example usage: hll -d -i /tmp/in.hll"); usage(options); return; } } // return after deserialization if (fis != null && in != null) { long start = System.currentTimeMillis(); HyperLogLog deserializedHLL = HyperLogLogUtils.deserializeHLL(in); long end = System.currentTimeMillis(); System.out.println(deserializedHLL.toString()); System.out.println("Count after deserialization: " + deserializedHLL.count()); System.out.println("Deserialization time: " + (end - start) + " ms"); return; } // construct hll and serialize it if required HyperLogLog hll = HyperLogLog.builder().enableBitPacking(bitPack).enableNoBias(noBias).setEncoding(enc) .setNumHashBits(hb).setNumRegisterIndexBits(p).build(); if (br != null) { Set<String> hashset = new HashSet<String>(); String line; while ((line = br.readLine()) != null) { hll.addString(line); hashset.add(line); } n = hashset.size(); } else { Random rand = new Random(seed); for (int i = 0; i < n; i++) { if (unique < 0) { hll.addLong(rand.nextLong()); } else { int val = rand.nextInt(unique); hll.addLong(val); } } } long estCount = hll.count(); System.out.println("Actual count: " + n); System.out.println(hll.toString()); System.out.println("Relative error: " + HyperLogLogUtils.getRelativeError(n, estCount) + "%"); if (fos != null && out != null) { long start = System.currentTimeMillis(); HyperLogLogUtils.serializeHLL(out, hll); long end = System.currentTimeMillis(); System.out.println("Serialized hyperloglog to " + outFile); System.out.println("Serialized size: " + out.size() + " bytes"); System.out.println("Serialization time: " + (end - start) + " ms"); out.close(); } } catch (ParseException e) { System.err.println("Invalid parameter."); usage(options); } catch (NumberFormatException e) { System.err.println("Invalid type for parameter."); usage(options); } catch (FileNotFoundException e) { System.err.println("Specified file not found."); usage(options); } catch (IOException e) { System.err.println("Exception occured while reading file."); usage(options); } }
From source file:Server_socket.java
public static void main(String argv[]) throws Exception { int puerto = Integer.parseInt(argv[0]); //String username = argv[1]; //String password = argv[2]; String clientformat;/*from w ww. j a va 2s. c om*/ String clientdata; String clientresult; String clientresource; String url_login = "http://localhost:3000/users/sign_in"; ServerSocket welcomeSocket = new ServerSocket(puerto); /*HttpClient httpClient = new DefaultHttpClient(); HttpPost httpPost = new HttpPost(url_login); // Add your data List<NameValuePair> nameValuePairs = new ArrayList<NameValuePair>(2); // nameValuePairs.add(new BasicNameValuePair("utf8", Character.toString('\u2713'))); nameValuePairs.add(new BasicNameValuePair("username", username)); nameValuePairs.add(new BasicNameValuePair("password", password)); // nameValuePairs.add(new BasicNameValuePair("commit", "Sign in")); httpPost.setEntity(new UrlEncodedFormEntity(nameValuePairs)); // Execute HTTP Post Request HttpResponse response = httpClient.execute(httpPost); String ret = EntityUtils.tostring(response.getEntity()); System.out.println(ret); */ while (true) { Socket connectionSocket = welcomeSocket.accept(); BufferedReader inFromClient = new BufferedReader( new InputStreamReader(connectionSocket.getInputStream())); clientformat = inFromClient.readLine(); System.out.println("FORMAT: " + clientformat); BufferedReader inFromClient1 = new BufferedReader( new InputStreamReader(connectionSocket.getInputStream())); clientdata = inFromClient1.readLine(); System.out.println("DATA: " + clientdata); BufferedReader inFromClient2 = new BufferedReader( new InputStreamReader(connectionSocket.getInputStream())); clientresult = inFromClient2.readLine(); System.out.println("RESULT: " + clientresult); BufferedReader inFromClient3 = new BufferedReader( new InputStreamReader(connectionSocket.getInputStream())); clientresource = inFromClient3.readLine(); System.out.println("RESOURCE: " + clientresource); System.out.println("no pasas de aqui"); String url = "http://localhost:3000/" + clientresource + "/" + clientdata + "." + clientformat; System.out.println(url); try (InputStream is = new URL(url).openStream()) { BufferedReader rd = new BufferedReader(new InputStreamReader(is, Charset.forName("UTF-8"))); StringBuilder sb = new StringBuilder(); int cp; while ((cp = rd.read()) != -1) { sb.append((char) cp); } String stb = sb.toString(); System.out.println("estas aqui"); String output = null; String jsonText = stb; output = jsonText.replace("[", "").replace("]", ""); JSONObject json = new JSONObject(output); System.out.println(json.toString()); System.out.println("llegaste al final"); DataOutputStream outToClient = new DataOutputStream(connectionSocket.getOutputStream()); outToClient.writeBytes(json.toString()); } connectionSocket.close(); } }
From source file:com.payne.test.StringTest.java
public static void main(String[] args) throws IOException { // String a = "160504185452148809-1"; // a = a.split("-")[0]; // System.out.println(a); // Random r = new Random(); // long a = 0l; // for(int i=0;i<10000;i++){ // a = r.nextInt(5001) + 5000; // if(a==5000){ // System.out.println(a); // } // }//w ww .j a v a 2 s. c om /** * ???? */ // List<Integer> numberList = new ArrayList<>(); // numberList.add(1); // numberList.add(2); // Integer[] numbers = numberList.toArray(new Integer[numberList.size()]); //// int[] numbers = new int[]{1,2}; // // System.out.println(new Integer[]{}.length==0?0:1); // // Student s = new Student(); // s.sumUp(new Integer[]{}.length==0?numbers:new Integer[]{1}); // s.sumUp(numbers); // Parent p = null; // Parent p2 = new Parent(new Student(5)); // // Student s = new Student(); // p = s.print(p); // System.out.println(p==null?0:1); // System.out.println(p.getAge()); // System.out.println(p.getStudent().getAge()); // int ai = 0; // for(int i=0;i<2;i++){ // int b = 0; // int b_grow = 0; // for(int j=0;j<5;j++){ // b += new Random().nextInt(5); // } // // } // // // System.out.println(UUID.randomUUID().toString()); // int a = 1; // a = doAdd(a); // System.out.println(a); Pattern p = Pattern.compile("^\\d{1,9}(.\\d{1,2})?$"); Matcher m = p.matcher("666666541.13"); boolean b = m.matches(); System.out.println(b); // System.out.println(-2>>4); // BigDecimal b = new BigDecimal(100.50); // System.out.println(b.toString()); // indexOf ?? // String a = "?"; // // String[] split = a.split("?"); // if(a.indexOf("?")>-1){ // System.out.println("111"); // } // for(String s: split){ // System.out.println(s); // } // MapTestObject mto = new MapTestObject(); // mto.setOrderType(OrderType.TWO); // // String str = "\":\""; // System.out.println(str); // String a = ","; //// // String[] splits = a.split("."); // List<String> asList = Arrays.asList(splits); // String last = ""; // String last = ""; // String last = ""; // if (!asList.isEmpty()) { // if (asList.indexOf(last) > -1) { // int indexOf = asList.indexOf(last); // if (indexOf + 1 >= asList.size()) { // System.out.println("?"); // }else{ // String next = asList.get(indexOf + 1); // if (OTHERStringUtils.isEmpty(next)) { // System.out.println("?"); // } else { // System.out.println("?" + next); // } // } // } else { // System.out.println("?"); // } // } else { // System.out.println("??"); // } // System.out.println("?\",\""); // String a ="1123"; // a = StringUtils.substring(a, 0, a.length()-1); // System.out.println("a="+a); // int a = 0x12345678; ByteArrayOutputStream baos = new ByteArrayOutputStream(); DataOutputStream dos = new DataOutputStream(baos); dos.writeInt(a); System.out.println(Integer.toHexString(a)); byte[] c = baos.toByteArray(); for (int i = 0; i < 4; i++) { System.out.println(Integer.toHexString(c[i])); } }
From source file:cn.jumper.study.http.ClientFormLogin.java
public static void main(String[] args) throws Exception { BasicCookieStore cookieStore = new BasicCookieStore(); CloseableHttpClient httpclient = HttpClients.custom().setDefaultCookieStore(cookieStore).build(); HttpHost proxy = new HttpHost("192.168.10.3", 8080, "http"); RequestConfig config = RequestConfig.custom().setProxy(proxy).build(); try {/*from w ww .j a va 2 s. c o m*/ HttpGet httpget = new HttpGet("http://www.ksf-food.com/admin/Login.asp"); httpget.setConfig(config); CloseableHttpResponse response1 = httpclient.execute(httpget); try { HttpEntity entity = response1.getEntity(); System.out.println("Login form get: " + response1.getStatusLine()); EntityUtils.consume(entity); System.out.println("Initial set of cookies:"); List<Cookie> cookies = cookieStore.getCookies(); if (cookies.isEmpty()) { System.out.println("None"); } else { for (int i = 0; i < cookies.size(); i++) { System.out.println("- " + cookies.get(i).toString()); } } } finally { response1.close(); } String code = ""; try { HttpUriRequest httpgetCode = RequestBuilder.get() .setUri("http://www.ksf-food.com/admin/inc/checkcode.asp").setConfig(config).build(); /* * HttpGet httpgetCode = new HttpGet( * "http://www.qufuev.com/admin/inc/checkcode.asp"); * httpgetCode.setConfig(config); */ System.out.println("Executing request " + httpgetCode.getRequestLine()); System.out.println("========================================================"); System.out.println("==httpget header =="); for (Header header : httpgetCode.getAllHeaders()) { System.out.println(header.getName() + ":" + header.getValue()); } System.out.println("==httpget header =="); ResponseHandler<String> responseHandler = new ResponseHandler<String>() { public String handleResponse(final HttpResponse response) throws ClientProtocolException, IOException { int status = response.getStatusLine().getStatusCode(); if (status >= 200 && status < 300) { HttpEntity entity = response.getEntity(); System.out.println("==respons header =="); for (Header header : response.getAllHeaders()) { System.out.println(header.getName() + ":" + header.getValue()); } System.out.println("==respons header =="); String fileName = System.currentTimeMillis() + ""; DataOutputStream dataOutputStream = new DataOutputStream( new FileOutputStream("d://test//e3//" + fileName + ".jpg")); dataOutputStream.write(EntityUtils.toByteArray(entity)); dataOutputStream.close(); return ImageTest.getAllOcr("d://test//e3//" + fileName + ".jpg"); } else { throw new ClientProtocolException("Unexpected response status: " + status); } } }; code = httpclient.execute(httpgetCode, responseHandler); System.out.println("ClientFormLogin.main()-CheckCode:" + code); System.out.println("----------------------------------------"); } catch (IOException e) { e.printStackTrace(); } HttpUriRequest login = RequestBuilder.post() .setUri(new URI("http://www.ksf-food.com/admin/Admin_ChkLogin.asp")) .addParameter("UserName", "username").addParameter("Password", "password") .addParameter("CheckCode", code).setConfig(config).build(); System.out.println("========================================================"); System.out.println("==httpget header =="); for (Header header : login.getAllHeaders()) { System.out.println(header.getName() + ":" + header.getValue()); } CloseableHttpResponse response2 = httpclient.execute(login); try { HttpEntity entity = response2.getEntity(); System.out.println("Login form post: " + response2.getStatusLine()); // EntityUtils.consume(entity); System.out.println("ClientFormLogin.main():\\n" + EntityUtils.toString(entity, "GBK")); System.out.println("Post logon cookies:"); List<Cookie> cookies = cookieStore.getCookies(); if (cookies.isEmpty()) { System.out.println("None"); } else { for (int i = 0; i < cookies.size(); i++) { System.out.println("- " + cookies.get(i).toString()); } } } finally { response2.close(); } } finally { httpclient.close(); } }
From source file:MySpaceParser.java
public static void main(String[] args) throws Exception { out = new DataOutputStream(new FileOutputStream("D:\\myspace.txt")); MySpaceParser sParser = new MySpaceParser(); sParser.parseHtmlData();//from w w w .j a v a 2s . c o m sParser.writeToFile(); }
From source file:org.wso2.charon3.samples.user.sample01.CreateUserSample.java
public static void main(String[] args) { try {//from w ww. j a va2 s. c o m String url = "http://localhost:8080/scim/v2/Users"; URL obj = new URL(url); HttpURLConnection con = (HttpURLConnection) obj.openConnection(); // Setting basic post request con.setRequestMethod("POST"); con.setRequestProperty("Content-Type", "application/scim+json"); // Send post request con.setDoOutput(true); DataOutputStream wr = new DataOutputStream(con.getOutputStream()); wr.writeBytes(createRequestBody); wr.flush(); wr.close(); int responseCode = con.getResponseCode(); BufferedReader in; if (responseCode == HttpURLConnection.HTTP_CREATED) { // success in = new BufferedReader(new InputStreamReader(con.getInputStream())); } else { in = new BufferedReader(new InputStreamReader(con.getErrorStream())); } String inputLine; StringBuffer response = new StringBuffer(); while ((inputLine = in.readLine()) != null) { response.append(inputLine); } in.close(); //printing result from response System.out.println("Response Code : " + responseCode); System.out.println("Response Message : " + con.getResponseMessage()); System.out.println("Response Content : " + response.toString()); } catch (ProtocolException e) { e.printStackTrace(); } catch (MalformedURLException e) { e.printStackTrace(); } catch (IOException e) { e.printStackTrace(); } }
From source file:net.java.sen.tools.MkSenDic.java
/** * Build sen dictionary./* w ww. j a v a 2 s . co m*/ * * @param args * custom dictionary files. see dic/build.xml. */ public static void main(String args[]) { ResourceBundle rb = ResourceBundle.getBundle("dictionary"); DictionaryMaker dm1 = new DictionaryMaker(); DictionaryMaker dm2 = new DictionaryMaker(); DictionaryMaker dm3 = new DictionaryMaker(); // 1st field information of connect file. Vector rule1 = new Vector(); // 2nd field information of connect file. Vector rule2 = new Vector(); // 3rd field information of connect file. Vector rule3 = new Vector(); // 4th field information of connect file. // this field shows cost of morpheme connection // [size3*(x3*size2+x2)+x1] // [size3*(Attr1*size2+Attr2)+Attl] short score[] = new short[20131]; long start = System.currentTimeMillis(); // ///////////////////////////////////////// // // Step1. Loading connetion file. // log.info("(1/7): reading connection matrix ... "); try { log.info("connection file = " + rb.getString("text_connection_file")); log.info("charset = " + rb.getString("dic.charset")); CSVParser csvparser = new CSVParser(new FileInputStream(rb.getString("text_connection_file")), rb.getString("dic.charset")); String t[]; int line = 0; while ((t = csvparser.nextTokens()) != null) { if (t.length < 4) { log.warn("invalid line in " + rb.getString("text_connection_file") + ":" + line); log.warn(rb.getString("text_connection_file") + "may be broken."); break; } dm1.add(t[0]); rule1.add(t[0]); dm2.add(t[1]); rule2.add(t[1]); dm3.add(t[2]); rule3.add(t[2]); if (line == score.length) { score = resize(score); } score[line++] = (short) Integer.parseInt(t[3]); } // ///////////////////////////////////////// // // Step2. Building internal dictionary // log.info("(2/7): building type dictionary ... "); dm1.build(); dm2.build(); dm3.build(); // if you want check specified morpheme, you uncomment and modify // following line: /* * System.out.print("22="); dm3.getById(22); * System.out.print("368="); dm3.getById(368); * * System.out.println(dm3.getDicId("?????*,*,*,*,?")); * DictionaryMaker.debug = true; * System.out.println(dm3.getDicId("?????*,*,*,*,?")); * System.out.println(dm3.getDicIdNoCache("?????*,*,*,*,?")); */ } catch (IOException e) { e.printStackTrace(); System.exit(0); } // ------------------------------------------------- int size1 = dm1.size(); int size2 = dm2.size(); int size3 = dm3.size(); int ruleSize = rule1.size(); short matrix[] = new short[size1 * size2 * size3]; short default_cost = (short) Integer.parseInt(rb.getString("default_connection_cost")); // ///////////////////////////////////////// // // Step3. Writing Connection Matrix // log.info("(3/7): writing conection matrix (" + size1 + " x " + size2 + " x " + size3 + " = " + size1 * size2 * size3 + ") ..."); for (int i = 0; i < (int) (size1 * size2 * size3); i++) matrix[i] = default_cost; for (int i = 0; i < ruleSize; i++) { Vector r1 = dm1.getRuleIdList((String) rule1.get(i)); Vector r2 = dm2.getRuleIdList((String) rule2.get(i)); Vector r3 = dm3.getRuleIdList((String) rule3.get(i)); for (Iterator i1 = r1.iterator(); i1.hasNext();) { int ii1 = ((Integer) i1.next()).intValue(); for (Iterator i2 = r2.iterator(); i2.hasNext();) { int ii2 = ((Integer) i2.next()).intValue(); for (Iterator i3 = r3.iterator(); i3.hasNext();) { int ii3 = ((Integer) i3.next()).intValue(); int pos = size3 * (size2 * ii1 + ii2) + ii3; matrix[pos] = score[i]; } } } } try { DataOutputStream out = new DataOutputStream( new BufferedOutputStream(new FileOutputStream(rb.getString("matrix_file")))); out.writeShort(size1); out.writeShort(size2); out.writeShort(size3); for (int i1 = 0; i1 < size1; i1++) for (int i2 = 0; i2 < size2; i2++) for (int i3 = 0; i3 < size3; i3++) { out.writeShort(matrix[size3 * (size2 * i1 + i2) + i3]); // if (matrix[size3 * (size2 * i1 + i2) + i3] != // default_cost) { // } } out.close(); } catch (IOException e) { e.printStackTrace(); System.exit(0); } matrix = null; score = null; // ------------------------------------------------- int pos_start = Integer.parseInt(rb.getString("pos_start")); int pos_size = Integer.parseInt(rb.getString("pos_size")); int di = 0; int offset = 0; ArrayList dicList = new ArrayList(); // ///////////////////////////////////////// // // Step4. Reading Morpheme Information // log.info("(4/7): reading morpheme information ... "); String t = null; String[] csv = null; try { // writer for feature file. BufferedWriter bw = new BufferedWriter(new OutputStreamWriter( new FileOutputStream(rb.getString("pos_file")), rb.getString("sen.charset"))); log.info("load dic: " + rb.getString("text_dic_file")); BufferedReader dicStream = null; int custom_dic = -1; if (args.length == 0) { dicStream = new BufferedReader(new InputStreamReader( new FileInputStream(rb.getString("text_dic_file")), rb.getString("dic.charset"))); } else { custom_dic = 0; dicStream = new BufferedReader( new InputStreamReader(new FileInputStream(args[custom_dic]), rb.getString("dic.charset"))); } int line = 0; CSVData key_b = new CSVData(); CSVData pos_b = new CSVData(); while (true) { t = dicStream.readLine(); if (t == null) { dicStream.close(); custom_dic++; if (args.length == custom_dic) { break; } else { // read custum dictionary log.info("load dic: " + "args[custum_dic]"); dicStream = new BufferedReader(new InputStreamReader(new FileInputStream(args[custom_dic]), rb.getString("dic.charset"))); } continue; } CSVParser parser = new CSVParser(t); csv = parser.nextTokens(); if (csv.length < (pos_size + pos_start)) { throw new RuntimeException("format error:" + t); } key_b.clear(); pos_b.clear(); for (int i = pos_start; i < (pos_start + pos_size - 1); i++) { key_b.append(csv[i]); pos_b.append(csv[i]); } key_b.append(csv[pos_start + pos_size - 1]); pos_b.append(csv[pos_start + pos_size - 1]); for (int i = pos_start + pos_size; i < (csv.length - 1); i++) { pos_b.append(csv[i]); } pos_b.append(csv[csv.length - 1]); CToken token = new CToken(); token.rcAttr2 = (short) dm1.getDicId(key_b.toString()); token.rcAttr1 = (short) dm2.getDicId(key_b.toString()); token.lcAttr = (short) dm3.getDicId(key_b.toString()); token.posid = 0; token.posID = offset; token.length = (short) csv[0].length(); token.cost = (short) Integer.parseInt(csv[1]); dicList.add(new PairObject(csv[0], token)); byte b[] = pos_b.toString().getBytes(rb.getString("sen.charset")); offset += (b.length + 1); String pos_b_str = pos_b.toString(); bw.write(pos_b_str, 0, pos_b_str.length()); // bw.write(b, 0, b.length); bw.write(0); if (++di % 50000 == 0) log.info("" + di + "... "); } bw.close(); // ----end of writing feature.cha ---- } catch (Exception e) { log.error("Error: " + t); e.printStackTrace(); System.exit(1); } rule1 = null; rule2 = null; rule3 = null; // ///////////////////////////////////////// // // Step5. Sort lexs and write to file // log.info("(5/7): sorting lex... "); int value[] = new int[dicList.size()]; char key[][] = new char[dicList.size()][]; int spos = 0; int dsize = 0; int bsize = 0; String prev = ""; Collections.sort(dicList); // ///////////////////////////////////////// // // Step6. Writing Token Information // log.info("(6/7): writing token... "); try { // writer for token file. DataOutputStream out = new DataOutputStream( new BufferedOutputStream(new FileOutputStream(rb.getString("token_file")))); // writing 'bos' and 'eos' and 'unknown' token. CToken token = new CToken(); token.rcAttr2 = (short) dm1.getDicId(rb.getString("bos_pos")); token.rcAttr1 = (short) dm2.getDicId(rb.getString("bos_pos")); token.lcAttr = (short) dm3.getDicId(rb.getString("bos_pos")); token.write(out); token.rcAttr2 = (short) dm1.getDicId(rb.getString("eos_pos")); token.rcAttr1 = (short) dm2.getDicId(rb.getString("eos_pos")); token.lcAttr = (short) dm3.getDicId(rb.getString("eos_pos")); token.write(out); token.rcAttr2 = (short) dm1.getDicId(rb.getString("unknown_pos")); token.rcAttr1 = (short) dm2.getDicId(rb.getString("unknown_pos")); token.lcAttr = (short) dm3.getDicId(rb.getString("unknown_pos")); token.posID = -1; token.write(out); log.info("key size = " + key.length); for (int i = 0; i < key.length; i++) { String k = (String) ((PairObject) dicList.get(i)).key; if (!prev.equals(k) && i != 0) { key[dsize] = ((String) ((PairObject) dicList.get(spos)).key).toCharArray(); value[dsize] = bsize + (spos << 8); dsize++; bsize = 1; spos = i; } else { bsize++; } prev = (String) ((PairObject) dicList.get(i)).key; ((CToken) (((PairObject) dicList.get(i)).value)).write(out); } out.flush(); out.close(); } catch (Exception e) { e.printStackTrace(); System.exit(1); } key[dsize] = ((String) ((PairObject) dicList.get(spos)).key).toCharArray(); value[dsize] = bsize + (spos << 8); dsize++; dm1 = null; dm2 = null; dm3 = null; dicList = null; // ///////////////////////////////////////// // // Step7. Build Double Array // log.info("(7/7): building Double-Array (size = " + dsize + ") ..."); DoubleArrayTrie da = new DoubleArrayTrie(); da.build(key, null, value, dsize); try { da.save(rb.getString("double_array_file")); } catch (Exception e) { e.printStackTrace(); } log.info("total time = " + (System.currentTimeMillis() - start) / 1000 + "[ms]"); }