Java tutorial
/** * Copyright 2000-2009 DFKI GmbH. * All Rights Reserved. Use is subject to license terms. * * This file is part of MARY TTS. * * MARY TTS is free software: you can redistribute it and/or modify * it under the terms of the GNU Lesser General Public License as published by * the Free Software Foundation, version 3 of the License. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public License * along with this program. If not, see <http://www.gnu.org/licenses/>. * */ package marytts.tools.voiceimport; import java.io.BufferedReader; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.FileReader; import java.io.FileWriter; import java.io.IOException; import java.io.InputStreamReader; import java.io.OutputStreamWriter; import java.io.PrintWriter; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; import java.util.SortedMap; import java.util.StringTokenizer; import java.util.TreeMap; import java.util.TreeSet; import java.util.regex.Matcher; import java.util.regex.Pattern; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.xpath.XPath; import javax.xml.xpath.XPathConstants; import javax.xml.xpath.XPathFactory; import marytts.client.MaryClient; import marytts.exceptions.MaryConfigurationException; import marytts.fst.FSTLookup; import marytts.modules.phonemiser.Allophone; import marytts.modules.phonemiser.AllophoneSet; import marytts.server.Mary; import marytts.util.io.FileUtils; import org.apache.commons.collections.set.ListOrderedSet; import org.w3c.dom.Document; import org.w3c.dom.Element; import org.w3c.dom.NodeList; /** * Automatic Labelling using HTK labeller * * @author Sathish Chandra Pammi * @author Fabio Tesser * * Fabio Tesser has fixed some HTK procedures and he has added the managing of virtual pauses (so called short pauses) after every word. * These pauses if detected will notified and it should be check if necessary to force the punctuation on the original text phrase in order to have a more coherent prosody. * Moreover in the initial phase, 1 mixture is used for each status; then the mixtures number is iteratively increased until a specific likelihood threshold is reached */ public class HTKLabeler extends VoiceImportComponent { private DatabaseLayout db; //private File rootDir; private File htk; private String voicename; private String outputDir; protected String xmlExt = ".xml"; protected String labExt = ".lab"; protected MaryClient mary; private int progress = -1; private String locale; protected String maryInputType; protected String maryOutputType; protected int percent = 0; protected File promtallophonesDir; Collection<String> HTKdictionary; Collection<String> Totaldictionary; protected FSTLookup lexicon; protected AllophoneSet allophoneSet; protected int MAX_ITERATIONS = 150; protected int MAX_SP_ITERATION = 10; // iteration when intra word forced pauses (ssil) are inserted protected int MAX_VP_ITERATION = 35; // iteration when virtual pauses (sp) are inserted protected int MAX_FA_ITERATION = 50; // iteration when forced alignement with multiple pronunciation is done protected int MAX_MIX_ITERATION = 70; // iteration when mixtures are increased protected int noIterCompleted = 0; public final String HTDIR = "HTKLabeler.htDir"; public final String HTKDIR = "HTKLabeler.htkDir"; public final String OUTLABDIR = "HTKLabeler.outputLabDir"; public final String MAXITER = "HTKLabeler.maxNoOfIterations"; public String PROMPTALLOPHONESDIR = "HTKLabeler.promtallophonesDir"; public String MAXSPITER = "HTKLabeler.maxshortPauseIteration"; private String HTK_SO = "-A -D -V -T 1"; //Main HTK standard Options HTK_SO private String Extract_FEAT = "MFCC_0"; //MFCC_E private String Train_FEAT = "MFCC_0_D_A"; //MFCC_E_D_A private int Train_VECTSIZE = 13 * 3; //13; //13 without D_A; 13*3 with D_A private int NUMStates = 5; private int[] num_mixtures_for_state = { 2, 1, 2 }; private int[] current_number_of_mixtures = { 1, 1, 1 }; //this is the starting number of mixtures (must be all ones) private ArrayList<Double> logProbFrame_array = new ArrayList<Double>(); private ArrayList<Double> epsilon_array = new ArrayList<Double>(); private int PHASE_NUMBER = 0; private double[] epsilon_PHASE = { 0.2, 0.05, 0.01, 0.0005, 0.0001 }; // 0 1 2 3 4 public final String getName() { return "HTKLabeler"; } public SortedMap getDefaultProps(DatabaseLayout db) { this.db = db; if (props == null) { props = new TreeMap(); String htkdir = System.getProperty("HTKDIR"); String phoneXml; locale = db.getProp(db.LOCALE); if (htkdir == null) { htkdir = "/usr/bin/"; } props.put(HTKDIR, htkdir); props.put(HTDIR, db.getProp(db.ROOTDIR) + System.getProperty("file.separator") + "htk" + System.getProperty("file.separator")); props.put(PROMPTALLOPHONESDIR, db.getProp(db.ROOTDIR) + System.getProperty("file.separator") + "prompt_allophones" + System.getProperty("file.separator")); props.put(OUTLABDIR, db.getProp(db.ROOTDIR) + System.getProperty("file.separator") + "htk" + System.getProperty("file.separator") + "lab" + System.getProperty("file.separator")); props.put(MAXITER, Integer.toString(MAX_ITERATIONS)); props.put(MAXSPITER, Integer.toString(MAX_SP_ITERATION)); } return props; } protected void setupHelp() { props2Help = new TreeMap(); props2Help.put(HTKDIR, "directory containing the HTK binary files of HTK, i.e (/usr/local/bin/)"); props2Help.put(HTDIR, "directory containing all files used for training and labeling. Will be created if it does not exist."); props2Help.put(PROMPTALLOPHONESDIR, "directory containing the prompt allophones files."); props2Help.put(OUTLABDIR, "Directory to store generated labels from HTK."); //props2Help.put(INITHTKDIR,"If you provide a path to previous HTK Directory, Models will intialize with those models. other wise HTK Models will build with Flat-Start Initialization"); //props2Help.put(RETRAIN,"true - Do re-training by initializing with given models. false - Do just Decoding"); props2Help.put(MAXITER, "Maximum number of iterations used for training"); props2Help.put(MAXSPITER, "Iteration number at which short-pause model need to insert."); } public void initialiseComp() { //dictionary = new TreeMap<String, TreeMap<String,String>>(); // Unused? HTKdictionary = new TreeSet<String>(); Totaldictionary = new TreeSet<String>(); promtallophonesDir = new File(getProp(PROMPTALLOPHONESDIR)); if (!promtallophonesDir.exists()) { System.out.print(PROMPTALLOPHONESDIR + " " + getProp(PROMPTALLOPHONESDIR) + " does not exist; "); if (!promtallophonesDir.mkdir()) { throw new Error("Could not create PROMPTALLOPHONESDIR"); } System.out.print("Created successfully.\n"); } } /** * Do the computations required by this component. * * @return true on success, false on failure */ public boolean compute() throws Exception { File htkFile = new File(getProp(HTKDIR) + File.separator + "HInit"); if (!htkFile.exists()) { throw new IOException( "HTK path setting is wrong. Because file " + htkFile.getAbsolutePath() + " does not exist"); } MAX_ITERATIONS = Integer.valueOf((getProp(MAXITER))); MAX_SP_ITERATION = Integer.valueOf((getProp(MAXSPITER))); System.out.println("Preparing voice database for labelling using HTK :"); //get the voicename voicename = db.getProp(db.VOICENAME); //make new directories htk and etc htk = new File(getProp(HTDIR)); // get the output directory of files used by HTK outputDir = htk.getAbsolutePath() + "/etc"; allophoneSet = AllophoneSet.getAllophoneSet(getClass().getResourceAsStream( "/marytts/language/" + locale.toString() + "/lexicon/allophones." + locale.toString() + ".xml"), "allophones." + locale.toString() + ".xml"); // part 1: HTK basic setup and create required files // setup the HTK directory System.out.println("Setting up HTK directory ..."); setup(); System.out.println(" ... done."); // create required files for HTK createRequiredFiles(); // creating phone dictionary. phone to phone mapping createPhoneDictionary(); // Extract phone sequence from prompt_allophones files getPhoneSequence(); saveHTKWordDictionary(); saveTotalWordDictionary(); // This is necessary to remove multiple sp: TODO: implement a loop and check the result delete_multiple_sp_in_PhoneMLFile( getProp(HTDIR) + File.separator + "etc" + File.separator + "htk.phones3.mlf", getProp(HTDIR) + File.separator + "etc" + File.separator + "htk.phones4.mlf"); delete_multiple_sp_in_PhoneMLFile( getProp(HTDIR) + File.separator + "etc" + File.separator + "htk.phones4.mlf", getProp(HTDIR) + File.separator + "etc" + File.separator + "htk.phones5.mlf"); delete_multiple_sp_in_PhoneMLFile( getProp(HTDIR) + File.separator + "etc" + File.separator + "htk.phones5.mlf", getProp(HTDIR) + File.separator + "etc" + File.separator + "htk.phones6.mlf"); delete_multiple_sp_in_PhoneMLFile( getProp(HTDIR) + File.separator + "etc" + File.separator + "htk.phones6.mlf", getProp(HTDIR) + File.separator + "etc" + File.separator + "htk.phones7.mlf"); delete_multiple_sp_in_PhoneMLFile( getProp(HTDIR) + File.separator + "etc" + File.separator + "htk.phones7.mlf", getProp(HTDIR) + File.separator + "etc" + File.separator + "htk.phones3.mlf"); delete_multiple_sp_in_PhoneMLFile( getProp(HTDIR) + File.separator + "etc" + File.separator + "htk.words3.mlf", getProp(HTDIR) + File.separator + "etc" + File.separator + "htk.words4.mlf"); delete_multiple_sp_in_PhoneMLFile( getProp(HTDIR) + File.separator + "etc" + File.separator + "htk.words4.mlf", getProp(HTDIR) + File.separator + "etc" + File.separator + "htk.words5.mlf"); delete_multiple_sp_in_PhoneMLFile( getProp(HTDIR) + File.separator + "etc" + File.separator + "htk.words5.mlf", getProp(HTDIR) + File.separator + "etc" + File.separator + "htk.words6.mlf"); delete_multiple_sp_in_PhoneMLFile( getProp(HTDIR) + File.separator + "etc" + File.separator + "htk.words6.mlf", getProp(HTDIR) + File.separator + "etc" + File.separator + "htk.words7.mlf"); delete_multiple_sp_in_PhoneMLFile( getProp(HTDIR) + File.separator + "etc" + File.separator + "htk.words7.mlf", getProp(HTDIR) + File.separator + "etc" + File.separator + "htk.words3.mlf"); //part 2: Feature Extraction using HCopy System.out.println("Feature Extraction:"); featureExtraction(); System.out.println("... Done."); //Part 3: Initialize Flat-start initialization System.out.println("HTK Training:"); initialiseHTKTrain(); createTrainFile(); //Part 4: training with HERest herestTraining(); System.out.println("... Done."); //Part 5: Force align with HVite /*System.out.println("HTK Align:"); hviteAligning(); System.out.println("... Done.");*/ //Part 5: Force align with HVite System.out.println("HTK Align:"); String logfile = "log_hviteMultiplePronunciationAligning_" + "final-full" + ".txt"; String labDir = getProp(HTDIR) + File.separator + "lab"; String alignedMlf = getProp(HTDIR) + File.separator + "aligned_full_final.mlf"; hviteMultiplePronunciationAligning("hmm" + "-final", alignedMlf, false, labDir, true, logfile, false); logfile = "log_hviteMultiplePronunciationAligning_" + "final-phones" + ".txt"; alignedMlf = getProp(HTDIR) + File.separator + "aligned.mlf"; hviteMultiplePronunciationAligning("hmm" + "-final", alignedMlf, false, labDir, false, logfile, false); // write the lab files logfile = "log_hviteMultiplePronunciationAligning_" + "final-full2" + ".txt"; alignedMlf = ""; labDir = getProp(HTDIR) + File.separator + "htk-full" + File.separator + "wrd"; hviteMultiplePronunciationAligning("hmm" + "-final", alignedMlf, true, labDir, true, logfile, false); logfile = "log_hviteMultiplePronunciationAligning_" + "final-phones" + ".txt"; alignedMlf = ""; labDir = getProp(HTDIR) + File.separator + "htk-full" + File.separator + "lab"; hviteMultiplePronunciationAligning("hmm" + "-final", alignedMlf, true, labDir, false, logfile, false); // to compare with the original pronunciations () logfile = "log_hviteMultiplePronunciationAligning_" + "final-phones_compare" + ".txt"; alignedMlf = getProp(HTDIR) + File.separator + "aligned_compare.mlf"; labDir = "'*'"; hviteMultiplePronunciationAligning("hmm" + "-final", alignedMlf, false, labDir, false, logfile, true); delete_multiple_sp_in_PhoneMLFile(alignedMlf, alignedMlf + ".norepetition"); System.out.println( "To check the recognized multiple pronunciations it is possible to make a comparision between " + getProp(HTDIR) + File.separator + "etc" + File.separator + "htk.words_phones3.mlf" + " and " + alignedMlf + ".norepetition"); System.out.println("... Done."); //Part 6: Extra model statistics System.out.println("Generating Extra model statistics..."); htkExtraModels(); System.out.println("... Done."); //Part 6: Generate Labels in required format System.out.println("Generating Labels in required format..."); getProperLabelFormat(); System.out.println(" ... done."); System.out.println("Label file Generation Successfully completed using HTK !"); return true; } private void saveHTKWordDictionary() throws Exception { String dict0 = outputDir + File.separator + "htk.words0.dict"; String dict = outputDir + File.separator + "htk.words.dict"; PrintWriter wordDictOut = new PrintWriter(new FileOutputStream(new File(dict0))); HTKdictionary.add("sil sil"); HTKdictionary.add("ssil ssil"); HTKdictionary.add("sp sp"); Iterator<String> itr = HTKdictionary.iterator(); while (itr.hasNext()) { wordDictOut.println(itr.next()); } wordDictOut.flush(); wordDictOut.close(); String fileded = getProp(HTDIR) + File.separator + "config" + File.separator + "dict.ded"; PrintWriter dedDictOut = new PrintWriter(new FileOutputStream(new File(fileded))); dedDictOut.println("AS sp"); dedDictOut.println("MP sil sil sp"); dedDictOut.println("MP ssil ssil sp"); dedDictOut.println("MP sp sp sp"); dedDictOut.flush(); dedDictOut.close(); Runtime rtime = Runtime.getRuntime(); //get a shell Process process = rtime.exec("/bin/bash"); //get an output stream to write to the shell //when no sp use (-m)! String hdman = getProp(HTKDIR) + File.separator + "HDMan"; PrintWriter pw = new PrintWriter(new OutputStreamWriter(process.getOutputStream())); String cmd = "( cd " + getProp(HTDIR) + "; " + hdman + " -g " + fileded + " " + dict + " " + dict0 + "; exit )\n"; System.out.println(cmd); pw.println(cmd); pw.flush(); //shut down pw.close(); process.waitFor(); // check exit value if (process.exitValue() != 0) { BufferedReader errorReader = new BufferedReader(new InputStreamReader(process.getErrorStream())); throw new MaryConfigurationException(errorReader.readLine()); } } private void saveTotalWordDictionary() throws FileNotFoundException { // total dictionary String totaldict = outputDir + File.separator + "htk.mary.words0.dict"; PrintWriter wordTotalDictOut = new PrintWriter(new FileOutputStream(new File(totaldict))); Iterator<String> itr = Totaldictionary.iterator(); while (itr.hasNext()) { wordTotalDictOut.println(itr.next()); } wordTotalDictOut.flush(); wordTotalDictOut.close(); } /** * Setup the HTK directory * @throws IOException, InterruptedException * @throws MaryConfigurationException */ private void setup() throws IOException, InterruptedException, MaryConfigurationException { htk.mkdir(); File lab = new File(htk.getAbsolutePath() + "/lab"); //call setup of HTK in this directory Runtime rtime = Runtime.getRuntime(); //get a shell Process process = rtime.exec("/bin/bash"); //get an output stream to write to the shell PrintWriter pw = new PrintWriter(new OutputStreamWriter(process.getOutputStream())); //go to htk directory and setup Directory Structure pw.print("( cd " + htk.getAbsolutePath() + "; mkdir -p hmm" + "; mkdir -p etc" + "; mkdir -p feat" + "; mkdir -p config" + "; mkdir -p lab" + "; mkdir -p htk-full/lab" + "; mkdir -p htk-full/wrd" + "; exit )\n"); pw.flush(); //shut down pw.close(); process.waitFor(); // check exit value if (process.exitValue() != 0) { BufferedReader errorReader = new BufferedReader(new InputStreamReader(process.getErrorStream())); throw new MaryConfigurationException(errorReader.readLine()); } // TODO: temporary: at the moment fix path to load the dictionary lexicon = new FSTLookup(new FileInputStream( "/home/fabio/voice_building_5_0/marytts-it/marytts-lang-it/src/main/resources/marytts/language/it/lexicon/it_lexicon.fst"), "it_lexicon.fst"); /* System.out.print("Starting builtin MARY TTS..."); Mary.startup(); System.out.println(" MARY TTS started."); e poi */ } /** * Creating phone dictionary (one-one mapping) and lists * @throws Exception */ private void createPhoneDictionary() throws Exception { PrintWriter transLabelOut = new PrintWriter(new FileOutputStream( new File(getProp(HTDIR) + File.separator + "etc" + File.separator + "htk" + ".phone.dict"))); PrintWriter phoneListOut = new PrintWriter(new FileOutputStream( new File(getProp(HTDIR) + File.separator + "etc" + File.separator + "htk" + ".phone.list"))); PrintWriter phoneListOut1 = new PrintWriter(new FileOutputStream( new File(getProp(HTDIR) + File.separator + "etc" + File.separator + "htk" + ".phone2.list"))); PrintWriter phoneListOut2 = new PrintWriter(new FileOutputStream( new File(getProp(HTDIR) + File.separator + "etc" + File.separator + "htk" + ".phone3.list"))); String phoneSeq; //transLabelOut.println("#!MLF!#"); Set<String> phonesList = allophoneSet.getAllophoneNames(); Iterator<String> it = phonesList.iterator(); while (it.hasNext()) { //System.out.println(it.next()); String phon = it.next(); if (phon.equals("_")) { continue; //phon = "sp"; } phon = replaceTrickyPhones(phon); transLabelOut.println(phon + " " + phon); phoneListOut.println(phon); phoneListOut1.println(phon); phoneListOut2.println(phon); } transLabelOut.println("sil" + " " + "sil"); phoneListOut.println("sil"); phoneListOut1.println("sil"); phoneListOut2.println("sil"); transLabelOut.println("ssil" + " " + "ssil"); phoneListOut1.println("ssil"); phoneListOut2.println("ssil"); transLabelOut.println("sp" + " " + "sp"); phoneListOut2.println("sp"); // commented G End Word // commented G Start Word /*phoneListOut.println("GEW"); phoneListOut1.println("GEW"); phoneListOut2.println("GEW"); phoneListOut.println("GSW"); phoneListOut1.println("GSW"); phoneListOut2.println("GSW"); transLabelOut.println("sp"+" "+"GEW"); transLabelOut.println("sp"+" "+"GEW sp"); transLabelOut.println("sp"+" "+"sp GSW"); transLabelOut.println("sp"+" "+"GSW"); transLabelOut.println("sp"+" "+"GEW sp GSW");*/ transLabelOut.flush(); transLabelOut.close(); phoneListOut.flush(); phoneListOut.close(); phoneListOut1.flush(); phoneListOut1.close(); phoneListOut2.flush(); phoneListOut2.close(); } /** * Create all required files(config files and HMM prototypes) for HTK Training * @throws Exception */ private void createRequiredFiles() throws Exception { // Creating mkphones0.led file, which insert and delete pauses // FABIO TODO: check is it used? File file = new File(getProp(HTDIR) + File.separator + "config" + File.separator + "mkphone0.led"); PrintWriter pw = new PrintWriter(new FileWriter(file)); pw.println("EX"); pw.println("IS sil sil"); //pw.println("DE sp"); // Short pause modelling pw.flush(); pw.close(); // Creating mkphones1.led file, which delete multiple sp pauses file = new File(getProp(HTDIR) + File.separator + "config" + File.separator + "mkphone1.led"); pw = new PrintWriter(new FileWriter(file)); pw.println("ME sp sp sp"); pw.println("ME sil sil sp"); pw.println("ME sil sp sil"); pw.println("ME ssil ssil sp"); pw.println("ME ssil sp ssil"); pw.println("ME sil sil sil"); pw.flush(); pw.close(); // creating a HTK Feature Extraction config file file = new File(getProp(HTDIR) + File.separator + "config" + File.separator + "featEx.conf"); pw = new PrintWriter(new FileWriter(file)); pw.println("SOURCEFORMAT = WAV # Gives the format of speech files "); pw.println("TARGETKIND = " + Extract_FEAT + " #Identifier of the coefficients to use"); pw.println("WINDOWSIZE = 100000.0 # = 10 ms = length of a time frame"); pw.println("TARGETRATE = 50000.0 # = 5 ms = frame periodicity"); pw.println("NUMCEPS = 12 # Number of MFCC coeffs (here from c1 to c12)"); pw.println("USEHAMMING = T # Use of Hamming funtion for windowing frames"); pw.println("PREEMCOEF = 0.97 # Pre-emphasis coefficient"); pw.println("NUMCHANS = 26 # Number of filterbank channels"); pw.println("CEPFILTER = 22 # Length of ceptral filtering"); pw.println("ENORMALISE = F # Energy measure normalization (sentence level)"); pw.flush(); pw.close(); //creating a HTK Training initialise config file file = new File(getProp(HTDIR) + File.separator + "config" + File.separator + "htkTrain.conf"); pw = new PrintWriter(new FileWriter(file)); pw.println("TARGETKIND = " + Train_FEAT + " #Identifier of the coefficients to use"); pw.println("PARAMETERKIND = " + Train_FEAT + ""); pw.println("WINDOWSIZE = 100000.0 # = 10 ms = length of a time frame"); pw.println("TARGETRATE = 50000.0 # = 5 ms = frame periodicity"); pw.println("NUMCEPS = 12 # Number of MFCC coeffs (here from c1 to c12)"); pw.println("USEHAMMING = T # Use of Hamming funtion for windowing frames"); pw.println("PREEMCOEF = 0.97 # Pre-emphasis coefficient"); pw.println("NUMCHANS = 26 # Number of filterbank channels"); pw.println("CEPFILTER = 22 # Length of ceptral filtering"); pw.println("ENORMALISE = F # Energy measure normalization (sentence level)"); pw.flush(); pw.close(); // Create an input file to HTK for Feature Extraction file = new File(getProp(HTDIR) + File.separator + "etc" + File.separator + "featEx.list"); pw = new PrintWriter(new FileWriter(file)); for (int i = 0; i < bnl.getLength(); i++) { //System.out.println( " " + bnl.getName(i) ); String input = db.getProp(db.WAVDIR) + File.separator + bnl.getName(i) + db.getProp(db.WAVEXT); String output = getProp(HTDIR) + File.separator + "feat" + File.separator + bnl.getName(i) + ".mfcc"; pw.println(input + " " + output); } pw.flush(); pw.close(); // creating list of training files file = new File(getProp(HTDIR) + File.separator + "etc" + File.separator + "htkTrain.list"); pw = new PrintWriter(new FileWriter(file)); for (int i = 0; i < bnl.getLength(); i++) { //System.out.println( " " + bnl.getName(i) ); String mFile = getProp(HTDIR) + File.separator + "feat" + File.separator + bnl.getName(i) + ".mfcc"; pw.println(mFile); } pw.flush(); pw.close(); //creating a hmm protofile int vectorSize = Train_VECTSIZE; int numStates = NUMStates; if (num_mixtures_for_state.length != numStates - 2) { throw new RuntimeException("Mixture num_mixtures_for_state lenght does not correspond to numStates"); } file = new File(getProp(HTDIR) + File.separator + "config" + File.separator + "htk.proto"); pw = new PrintWriter(new FileWriter(file)); pw.println("<BeginHMM>"); pw.println("<NumStates> " + numStates + " <VecSize> " + vectorSize + " <" + Train_FEAT + ">"); for (int state = 2; state < numStates; state++) { pw.println("<State> " + state); //pw.println("<NumMixes> " + num_mixtures_for_state[state-2]); //for(int mix=1;mix<=num_mixtures_for_state[state-2];mix++){ //pw.println("<Mixture> " + mix + " " + 1.0/num_mixtures_for_state[state-2]); pw.println("<Mean> " + vectorSize); for (int j = 0; j < vectorSize; j++) { pw.print(" 0.0"); } pw.println(); pw.println("<Variance> " + vectorSize); for (int j = 0; j < vectorSize; j++) { pw.print(" 1.0"); } pw.println(); } //} pw.println("<TransP> " + numStates); pw.println("0.0 1.0 0.0 0.0 0.0"); pw.println("0.0 0.6 0.4 0.0 0.0"); pw.println("0.0 0.0 0.6 0.4 0.0"); pw.println("0.0 0.0 0.0 0.7 0.3"); pw.println("0.0 0.0 0.0 0.0 1.0"); pw.println("<EndHMM>"); pw.flush(); pw.close(); // Creating SSIL Silence modeling config file file = new File(getProp(HTDIR) + File.separator + "config" + File.separator + "sil.hed"); pw = new PrintWriter(new FileWriter(file)); pw.println("AT 2 4 0.2 {sil.transP}"); pw.println("AT 4 2 0.2 {sil.transP}"); //pw.println("AT 1 3 0.3 {ssil.transP}"); //pw.println("TI silst {sil.state[3],ssil.state[2]}"); pw.println("AT 2 4 0.2 {ssil.transP}"); pw.println("AT 4 2 0.2 {ssil.transP}"); // added tied states... pw.println("TI silst2 {sil.state[2],ssil.state[2]}"); pw.println("TI silst3 {sil.state[3],ssil.state[3]}"); pw.println("TI silst4 {sil.state[4],ssil.state[4]}"); pw.flush(); pw.close(); // Creating SP Silence modeling config file file = new File(getProp(HTDIR) + File.separator + "config" + File.separator + "sil_vp.hed"); pw = new PrintWriter(new FileWriter(file)); //sp 3 state case: //pw.println("AT 1 3 0.3 {sp.transP}"); //pw.println("TI ssilst {ssil.state[3],sp.state[2]}"); //sp 5 state case: pw.println("AT 1 5 0.3 {sp.transP}"); pw.println("TI ssilst2 {ssil.state[2],sp.state[2]}"); pw.println("TI ssilst3 {ssil.state[3],sp.state[3]}"); pw.println("TI ssilst4 {ssil.state[4],sp.state[4]}"); pw.flush(); pw.close(); } /** * delete sp repetition on htk.phones3.mlf * @throws Exception */ private void delete_multiple_sp_in_PhoneMLFile(String filein, String fileout) throws Exception { String hled = getProp(HTKDIR) + File.separator + "HLEd"; File htkFile = new File(hled); if (!htkFile.exists()) { throw new RuntimeException("File " + htkFile.getAbsolutePath() + " does not exist"); } //String phoneMLF3 = getProp(HTDIR)+File.separator // +"etc"+File.separator+"htk.phones3.mlf"; // String phoneMLFtmpin = getProp(HTDIR)+File.separator // +"etc"+File.separator+"htk.phones3_tmp_in.mlf"; //String phoneMLFtmpout = getProp(HTDIR)+File.separator // +"etc"+File.separator+"htk.phones3_tmp_out.mlf"; String mkphoneLED = getProp(HTDIR) + File.separator + "config" + File.separator + "mkphone1.led"; Runtime rtime = Runtime.getRuntime(); //get a shell Process process = rtime.exec("/bin/bash"); //get an output stream to write to the shell PrintWriter pw = new PrintWriter(new OutputStreamWriter(process.getOutputStream())); System.out.println("( " + hled + " -l '*' -i " + fileout + " " + mkphoneLED + " " + filein + "; exit )\n"); pw.print("( " + hled + " -l '*' -i " + fileout + " " + mkphoneLED + " " + filein //+"; " + "; exit )\n"); pw.flush(); //shut down pw.close(); process.waitFor(); process.exitValue(); } /** * create phone master label file (Not used?) * @throws Exception */ private void createPhoneMLFile() throws Exception { String hled = getProp(HTKDIR) + File.separator + "HLEd"; File htkFile = new File(hled); if (!htkFile.exists()) { throw new RuntimeException("File " + htkFile.getAbsolutePath() + " does not exist"); } String dict = getProp(HTDIR) + File.separator + "etc" + File.separator + "htk.dict"; String phoneMLF = getProp(HTDIR) + File.separator + "etc" + File.separator + "htk.phones.mlf"; String wordsMLF = getProp(HTDIR) + File.separator + "etc" + File.separator + "htk.words.mlf"; String mkphoneLED = getProp(HTDIR) + File.separator + "config" + File.separator + "mkphone0.led"; Runtime rtime = Runtime.getRuntime(); //get a shell Process process = rtime.exec("/bin/bash"); //get an output stream to write to the shell PrintWriter pw = new PrintWriter(new OutputStreamWriter(process.getOutputStream())); System.out.println("( " + hled + " -l '*' -d " + dict + " -i " + phoneMLF + " " + mkphoneLED + " " + wordsMLF + "; exit )\n"); pw.print("( " + hled + " -l '*' -d " + dict + " -i " + phoneMLF + " " + mkphoneLED + " " + wordsMLF //+"; " + "; exit )\n"); pw.flush(); //shut down pw.close(); process.waitFor(); // check exit value if (process.exitValue() != 0) { BufferedReader errorReader = new BufferedReader(new InputStreamReader(process.getErrorStream())); throw new MaryConfigurationException(errorReader.readLine()); } } /** * Feature Extraction for HTK Training * @throws Exception */ private void featureExtraction() throws Exception { String hcopy = getProp(HTKDIR) + File.separator + "HCopy"; File htkFile = new File(hcopy); if (!htkFile.exists()) { throw new RuntimeException("File " + htkFile.getAbsolutePath() + " does not exist"); } String configFile = getProp(HTDIR) + File.separator + "config" + File.separator + "featEx.conf"; String listFile = getProp(HTDIR) + File.separator + "etc" + File.separator + "featEx.list"; Runtime rtime = Runtime.getRuntime(); //get a shell Process process = rtime.exec("/bin/bash"); //get an output stream to write to the shell PrintWriter pw = new PrintWriter(new OutputStreamWriter(process.getOutputStream())); System.out.println("( cd " + getProp(HTDIR) + "; " + hcopy + " -T 1 -C " + configFile + " -S " + listFile + " > log_featureExtraction.txt" + "; exit )\n"); pw.print("( cd " + getProp(HTDIR) + "; " + hcopy + " -T 1 -C " + configFile + " -S " + listFile + " > log_featureExtraction.txt" + "; exit )\n"); pw.flush(); //shut down pw.close(); process.waitFor(); // check exit value if (process.exitValue() != 0) { BufferedReader errorReader = new BufferedReader(new InputStreamReader(process.getErrorStream())); throw new MaryConfigurationException(errorReader.readLine()); } } /** * Initialize HTK Training process * @throws Exception */ private void initialiseHTKTrain() throws Exception { String hcompv = getProp(HTKDIR) + File.separator + "HCompV"; File htkFile = new File(hcompv); if (!htkFile.exists()) { throw new RuntimeException("File " + htkFile.getAbsolutePath() + " does not exist"); } String configFile = getProp(HTDIR) + File.separator + "config" + File.separator + "htkTrain.conf"; String listFile = getProp(HTDIR) + File.separator + "etc" + File.separator + "htkTrain.list"; Runtime rtime = Runtime.getRuntime(); //get a shell Process process = rtime.exec("/bin/bash"); //get an output stream to write to the shell PrintWriter pw = new PrintWriter(new OutputStreamWriter(process.getOutputStream())); System.out.println("( cd " + getProp(HTDIR) + " ; mkdir hmm/hmm-dummy ; " + " mkdir hmm/hmm-final ; " + hcompv + " " + HTK_SO + " -C " + configFile + " -f 0.01 -m -S " + listFile + " -M " + getProp(HTDIR) + File.separator + "hmm/hmm-dummy " + getProp(HTDIR) + File.separator + "config" + File.separator + "htk.proto" + " > log_initialiseHTKTrain.txt" + "; exit )\n"); pw.print("( cd " + getProp(HTDIR) + " ; mkdir hmm/hmm-dummy ; " + " mkdir hmm/hmm-final ; " + hcompv + " " + HTK_SO + " -C " + configFile + " -f 0.01 -m -S " + listFile + " -M " + getProp(HTDIR) + File.separator + "hmm/hmm-dummy " + getProp(HTDIR) + File.separator + "config" + File.separator + "htk.proto" + " > log_initialiseHTKTrain.txt" + "; exit )\n"); pw.flush(); //shut down pw.close(); process.waitFor(); // check exit value if (process.exitValue() != 0) { BufferedReader errorReader = new BufferedReader(new InputStreamReader(process.getErrorStream())); throw new MaryConfigurationException(errorReader.readLine()); } } /** * Create HMMs for each phone from Global HMMs * @throws Exception */ private void createTrainFile() throws Exception { String script; String hmmDir = getProp(HTDIR) + File.separator + "hmm" + File.separator; /**TODO: * Replace below 'gawk' script with Java method. */ script = "mkdir hmm/hmm0\n" + "head -3 hmm/hmm-dummy/htk > hmm/hmm0/hmmdefs\n" + "for s in `cat etc/htk.phone.list`\n" + "do\n" + "echo \"~h \\\"$s\\\"\" >> hmm/hmm0/hmmdefs\n" + "gawk '/BEGINHMM/,/ENDHMM/ { print $0 }' hmm/hmm-dummy/htk >> hmm/hmm0/hmmdefs\n" + "done\n"; // creating list of training files File file = new File(getProp(HTDIR) + File.separator + "etc" + File.separator + "htkTrainScript.sh"); PrintWriter pw = new PrintWriter(new FileWriter(file)); pw.println(script); pw.flush(); pw.close(); Runtime rtime = Runtime.getRuntime(); //get a shell Process process = rtime.exec("/bin/bash"); //get an output stream to write to the shell pw = new PrintWriter(new OutputStreamWriter(process.getOutputStream())); System.out.println("( cd " + getProp(HTDIR) + "; sh etc" + File.separator + "htkTrainScript.sh" + " > log_htkTrainScript.txt" + "; exit )\n"); pw.print("( cd " + getProp(HTDIR) + "; sh etc" + File.separator + "htkTrainScript.sh" + " > log_htkTrainScript.txt" + "; exit )\n"); pw.flush(); //shut down pw.close(); process.waitFor(); // check exit value if (process.exitValue() != 0) { BufferedReader errorReader = new BufferedReader(new InputStreamReader(process.getErrorStream())); throw new MaryConfigurationException(errorReader.readLine()); } PrintWriter macroFile = new PrintWriter( new FileOutputStream(new File(hmmDir + "hmm0" + File.separator + "macros"))); macroFile.println("~o\n" + "<VecSize> 13\n" + "<" + Train_FEAT + ">"); macroFile.println( FileUtils.getFileAsString(new File(hmmDir + "hmm-dummy" + File.separator + "vFloors"), "ASCII")); macroFile.flush(); macroFile.close(); } // TODO: check why log is empty! /** * Flat-start initialization for automatic labeling * @throws Exception */ private void herestTraining() throws Exception { String herest = getProp(HTKDIR) + File.separator + "HERest"; String hhed = getProp(HTKDIR) + File.separator + "HHEd"; File htkFile = new File(herest); if (!htkFile.exists()) { throw new RuntimeException("File " + htkFile.getAbsolutePath() + " does not exist"); } String configFile = getProp(HTDIR) + File.separator + "config" + File.separator + "htkTrain.conf"; String hhedconf = getProp(HTDIR) + File.separator + "config" + File.separator + "sil.hed"; String hhedconf_vp = getProp(HTDIR) + File.separator + "config" + File.separator + "sil_vp.hed"; String trainList = getProp(HTDIR) + File.separator + "etc" + File.separator + "htkTrain.list"; String phoneList = getProp(HTDIR) + File.separator + "etc" + File.separator + "htk.phone.list"; String hmmDir = getProp(HTDIR) + File.separator + "hmm" + File.separator; String phoneMlf = getProp(HTDIR) + File.separator + "etc" + File.separator + "htk.phones.mlf"; int BEST_ITERATION = MAX_ITERATIONS; int SP_ITERATION = -1; int VP_ITERATION = -1; int FA_ITERATION = -1; int change_mix_iteration = -1; for (int iteration = 1; iteration <= MAX_ITERATIONS; iteration++) { System.out.println("Iteration number: " + iteration); File hmmItDir = new File(hmmDir + "hmm" + iteration); if (!hmmItDir.exists()) hmmItDir.mkdir(); Runtime rtime = Runtime.getRuntime(); //get a shell Process process = rtime.exec("/bin/bash"); //get an output stream to write to the shell PrintWriter pw = new PrintWriter(new OutputStreamWriter(process.getOutputStream())); if (PHASE_NUMBER == 0) { if (iteration == (SP_ITERATION + 1)) { phoneMlf = getProp(HTDIR) + File.separator + "etc" + File.separator + "htk.phones2.mlf"; phoneList = getProp(HTDIR) + File.separator + "etc" + File.separator + "htk.phone2.list"; System.out.println("( cd " + getProp(HTDIR) + "; " + hhed + " " + HTK_SO + " -H " + hmmDir + "hmm" + (iteration - 1) + File.separator + "macros" + " -H " + hmmDir + "hmm" + (iteration - 1) + File.separator + "hmmdefs" + " -M " + hmmDir + "hmm" + iteration + " " + hhedconf + " " + phoneList + " >> log_herestTraining_" + iteration + ".txt" + "; exit )\n"); pw.println("( cd " + getProp(HTDIR) + "; " + hhed + " " + HTK_SO + " -H " + hmmDir + "hmm" + (iteration - 1) + File.separator + "macros" + " -H " + hmmDir + "hmm" + (iteration - 1) + File.separator + "hmmdefs" + " -M " + hmmDir + "hmm" + iteration + " " + hhedconf + " " + phoneList + " >> log_herestTraining_" + iteration + ".txt" + "; exit )\n"); pw.flush(); //shut down pw.close(); process.waitFor(); // check exit value if (process.exitValue() != 0) { BufferedReader errorReader = new BufferedReader( new InputStreamReader(process.getErrorStream())); throw new MaryConfigurationException(errorReader.readLine()); } // copy of logProbFrame_array in current iteration logProbFrame_array.add(logProbFrame_array.get(iteration - 2)); epsilon_array.add(100000000.0); //now we enter in PHASE 1 PHASE_NUMBER = 1; System.out.println("Now we enter in PHASE:" + PHASE_NUMBER); continue; } // check epsilon_array if (iteration > 2) { if (epsilon_array.get(iteration - 2) < epsilon_PHASE[PHASE_NUMBER] || iteration == MAX_SP_ITERATION) { SP_ITERATION = iteration; insertShortPause(iteration); String oldMacro = hmmDir + "hmm" + (iteration - 1) + File.separator + "macros"; String newMacro = hmmDir + "hmm" + iteration + File.separator + "macros"; FileUtils.copy(oldMacro, newMacro); // copy of logProbFrame_array in current iteration logProbFrame_array.add(logProbFrame_array.get(iteration - 2)); epsilon_array.add(100000000.0); continue; } } } ///----------------- if (PHASE_NUMBER == 1) { if (iteration == (VP_ITERATION + 1)) { phoneMlf = getProp(HTDIR) + File.separator + "etc" + File.separator + "htk.phones3.mlf"; phoneList = getProp(HTDIR) + File.separator + "etc" + File.separator + "htk.phone3.list"; System.out.println("( cd " + getProp(HTDIR) + "; " + hhed + " " + HTK_SO + " -H " + hmmDir + "hmm" + (iteration - 1) + File.separator + "macros" + " -H " + hmmDir + "hmm" + (iteration - 1) + File.separator + "hmmdefs" + " -M " + hmmDir + "hmm" + iteration + " " + hhedconf_vp + " " + phoneList + " >> log_herestTraining_" + iteration + ".txt" + "; exit )\n"); pw.println("( cd " + getProp(HTDIR) + "; " + hhed + " " + HTK_SO + " -H " + hmmDir + "hmm" + (iteration - 1) + File.separator + "macros" + " -H " + hmmDir + "hmm" + (iteration - 1) + File.separator + "hmmdefs" + " -M " + hmmDir + "hmm" + iteration + " " + hhedconf_vp + " " + phoneList + " >> log_herestTraining_" + iteration + ".txt" + "; exit )\n"); pw.flush(); //shut down pw.close(); process.waitFor(); // check exit value if (process.exitValue() != 0) { BufferedReader errorReader = new BufferedReader( new InputStreamReader(process.getErrorStream())); throw new MaryConfigurationException(errorReader.readLine()); } // copy of logProbFrame_array in current iteration logProbFrame_array.add(logProbFrame_array.get(iteration - 2)); epsilon_array.add(100000000.0); //now we enter in PHASE 2 PHASE_NUMBER = 2; System.out.println("Now we enter in PHASE:" + PHASE_NUMBER); continue; } // check epsilon_array if (epsilon_array.get(iteration - 2) < epsilon_PHASE[PHASE_NUMBER] || iteration == MAX_VP_ITERATION) { VP_ITERATION = iteration; insertVirtualPauseThreeStates(iteration); String oldMacro = hmmDir + "hmm" + (iteration - 1) + File.separator + "macros"; String newMacro = hmmDir + "hmm" + iteration + File.separator + "macros"; FileUtils.copy(oldMacro, newMacro); // copy of logProbFrame_array in current iteration logProbFrame_array.add(logProbFrame_array.get(iteration - 2)); epsilon_array.add(100000000.0); continue; } } ///----------------- if (PHASE_NUMBER == 2) { if (iteration == (FA_ITERATION + 1)) { String logfile = "log_hviteMultiplePronunciationAligning_" + iteration + ".txt"; String labDir = "'*'";//getProp(HTDIR)+File.separator+"lab"; String alignedMlf = getProp(HTDIR) + File.separator + "aligned_words.mlf"; hviteMultiplePronunciationAligning("hmm" + (iteration - 1), alignedMlf, false, labDir, true, logfile, false); phoneMlf = getProp(HTDIR) + File.separator + "aligned_words.mlf"; System.out.println("Copy hmm" + (iteration - 1) + " in " + "hmm" + iteration); String oldMacro = hmmDir + "hmm" + (iteration - 1) + File.separator + "macros"; String newMacro = hmmDir + "hmm" + iteration + File.separator + "macros"; FileUtils.copy(oldMacro, newMacro); String oldHmmdefs = hmmDir + "hmm" + (iteration - 1) + File.separator + "hmmdefs"; String newHmmdefs = hmmDir + "hmm" + iteration + File.separator + "hmmdefs"; FileUtils.copy(oldHmmdefs, newHmmdefs); // copy of logProbFrame_array in current iteration logProbFrame_array.add(logProbFrame_array.get(iteration - 2)); epsilon_array.add(100000000.0); //now we enter in PHASE 3 PHASE_NUMBER = 3; System.out.println("Now we enter in PHASE:" + PHASE_NUMBER); continue; } // check epsilon_array if (epsilon_array.get(iteration - 2) < epsilon_PHASE[PHASE_NUMBER] || iteration == MAX_FA_ITERATION) { FA_ITERATION = iteration; System.out.println("Copy hmm" + (iteration - 1) + " in " + "hmm" + iteration); String oldMacro = hmmDir + "hmm" + (iteration - 1) + File.separator + "macros"; String newMacro = hmmDir + "hmm" + iteration + File.separator + "macros"; FileUtils.copy(oldMacro, newMacro); String oldHmmdefs = hmmDir + "hmm" + (iteration - 1) + File.separator + "hmmdefs"; String newHmmdefs = hmmDir + "hmm" + iteration + File.separator + "hmmdefs"; FileUtils.copy(oldHmmdefs, newHmmdefs); // copy of logProbFrame_array in current iteration logProbFrame_array.add(logProbFrame_array.get(iteration - 2)); epsilon_array.add(100000000.0); continue; } } ///----------------- if (PHASE_NUMBER == 3) { // check epsilon_array // the following change_mix_iteration + 2 is used to allow more than one re-estimation after insertion of new mixture // Because just after the insertion the delta can be negative if (((iteration != change_mix_iteration + 2) && (epsilon_array.get(iteration - 2) < epsilon_PHASE[PHASE_NUMBER])) || iteration == MAX_MIX_ITERATION) { System.out.println("Condition = true: " + "iteration=" + iteration + " change_mix_iteration=" + change_mix_iteration + " epsilon_array.get(iteration-2)=" + epsilon_array.get(iteration - 2) + " epsilon_PHASE[PHASE_NUMBER]=" + epsilon_PHASE[PHASE_NUMBER] + " MAX_MIX_ITERATION" + MAX_MIX_ITERATION); change_mix_iteration = iteration; MAX_MIX_ITERATION = -1; // Creating Increasing mixture config file dynamic iteration String hhedconf_mix = getProp(HTDIR) + File.separator + "config" + File.separator + "sil_mix_" + iteration + ".hed"; File file = new File(hhedconf_mix); PrintWriter hhed_conf_pw = new PrintWriter(new FileWriter(file)); //MU 3 {*.state[2].mix} Boolean need_other_updates = false; for (int state = 0; state < num_mixtures_for_state.length; state++) { if (current_number_of_mixtures[state] < num_mixtures_for_state[state]) { int wanted_mix = current_number_of_mixtures[state] + 1; int state_to_print = state + 2; hhed_conf_pw.println("MU " + wanted_mix + "{*.state[" + state_to_print + "].mix}"); current_number_of_mixtures[state] = wanted_mix; if (current_number_of_mixtures[state] < num_mixtures_for_state[state]) { need_other_updates = true; } } } if (!need_other_updates) { // copy of logProbFrame_array in current iteration //logProbFrame_array.add(logProbFrame_array.get(iteration-2)); //epsilon_array.add(100000000.0); //now we enter in PHASE 3 PHASE_NUMBER = PHASE_NUMBER + 1; System.out.println("Now we enter in PHASE:" + PHASE_NUMBER); String logfile = "log_hviteMultiplePronunciationAligning_" + iteration + ".txt"; String labDir = "'*'";//getProp(HTDIR)+File.separator+"lab"; String alignedMlf = getProp(HTDIR) + File.separator + "aligned_words.mlf"; hviteMultiplePronunciationAligning("hmm" + (iteration - 1), alignedMlf, false, labDir, true, logfile, false); phoneMlf = getProp(HTDIR) + File.separator + "aligned_words.mlf"; //continue; } hhed_conf_pw.flush(); hhed_conf_pw.close(); System.out.println("( cd " + getProp(HTDIR) + "; " + hhed + " " + HTK_SO + " -H " + hmmDir + "hmm" + (iteration - 1) + File.separator + "macros" + " -H " + hmmDir + "hmm" + (iteration - 1) + File.separator + "hmmdefs" + " -M " + hmmDir + "hmm" + iteration + " " + hhedconf_mix + " " + phoneList + " >> log_herestTraining_" + iteration + ".txt" + "; exit )\n"); pw.println("( cd " + getProp(HTDIR) + "; " + hhed + " " + HTK_SO + " -H " + hmmDir + "hmm" + (iteration - 1) + File.separator + "macros" + " -H " + hmmDir + "hmm" + (iteration - 1) + File.separator + "hmmdefs" + " -M " + hmmDir + "hmm" + iteration + " " + hhedconf_mix + " " + phoneList + " >> log_herestTraining_" + iteration + ".txt" + "; exit )\n"); pw.flush(); //shut down pw.close(); process.waitFor(); // check exit value if (process.exitValue() != 0) { BufferedReader errorReader = new BufferedReader( new InputStreamReader(process.getErrorStream())); throw new MaryConfigurationException(errorReader.readLine()); } // copy of logProbFrame_array in current iteration logProbFrame_array.add(logProbFrame_array.get(iteration - 2)); epsilon_array.add(100000000.0); continue; } } ///----------------- if (PHASE_NUMBER == 4) { // check epsilon_array if (((iteration != change_mix_iteration + 2) && (epsilon_array.get(iteration - 2) < epsilon_PHASE[PHASE_NUMBER])) || iteration == MAX_ITERATIONS) { int last = iteration - 1; int previus_last = iteration - 2; System.out.println( "Average log prob per frame has not beeen increased too much respect the previus iteration:"); System.out.println("Average log prob per frame at last HREST iteration (" + last + ")-> " + logProbFrame_array.get(iteration - 2)); System.out.println("Average log prob per frame at previus HREST iteration (" + previus_last + ")-> " + logProbFrame_array.get(iteration - 3)); System.out.println("Delta -> " + epsilon_array.get(iteration - 2)); System.out.println("Suggested Action -> stop the iterations."); if (logProbFrame_array.get(iteration - 3) > logProbFrame_array.get(iteration - 2)) { BEST_ITERATION = iteration - 2; } else { BEST_ITERATION = iteration - 1; } break; } } //Normal HEREST: System.out.println("( cd " + getProp(HTDIR) + "; " + herest + " " + HTK_SO + " -C " + configFile + " -I " + phoneMlf + " -t 250.0 150.0 1000.0" + " -S " + trainList + " -H " + hmmDir + "hmm" + (iteration - 1) + File.separator + "macros" + " -H " + hmmDir + "hmm" + (iteration - 1) + File.separator + "hmmdefs" + " -M " + hmmDir + "hmm" + iteration + " " + phoneList + " >> log_herestTraining_" + iteration + ".txt" + "; exit )\n"); pw.println("( cd " + getProp(HTDIR) + "; " + herest + " " + HTK_SO + " -C " + configFile + " -I " + phoneMlf + " -t 250.0 150.0 1000.0" + " -S " + trainList + " -H " + hmmDir + "hmm" + (iteration - 1) + File.separator + "macros" + " -H " + hmmDir + "hmm" + (iteration - 1) + File.separator + "hmmdefs" + " -M " + hmmDir + "hmm" + iteration + " " + phoneList + " >> log_herestTraining_" + iteration + ".txt" + "; exit )\n"); pw.flush(); //shut down pw.close(); process.waitFor(); // check exit value if (process.exitValue() != 0) { BufferedReader errorReader = new BufferedReader(new InputStreamReader(process.getErrorStream())); throw new MaryConfigurationException(errorReader.readLine()); } // update average_log_prob_per_frame and deltas check_average_log_prob_per_frame(iteration); System.out.println("Delta average log prob per frame to respect previus iteration-> " + epsilon_array.get(iteration - 1)); System.out.println("Current PHASE: " + PHASE_NUMBER); System.out.println("Current state and number of mixtures (for each phoneme): " + Arrays.toString(current_number_of_mixtures)); System.out.println("---------------------------------------"); } System.out.println("***********\n"); System.out.println("BEST ITERATION: " + BEST_ITERATION); System.out.println("COPYNING BEST ITERATION FILES IN hmm-final directory"); System.out.println("logProbFrame_array:" + logProbFrame_array.toString()); System.out.println("epsilon_array:" + epsilon_array.toString()); System.out.println("***********\n"); String oldMacro = hmmDir + "hmm" + BEST_ITERATION + File.separator + "macros"; String newMacro = hmmDir + "hmm-final" + File.separator + "macros"; FileUtils.copy(oldMacro, newMacro); String oldHmmdefs = hmmDir + "hmm" + BEST_ITERATION + File.separator + "hmmdefs"; String newHmmdefs = hmmDir + "hmm-final" + File.separator + "hmmdefs"; FileUtils.copy(oldHmmdefs, newHmmdefs); } private void check_average_log_prob_per_frame(int iteration) throws IOException { // TODO Auto-generated method stub String filename = getProp(HTDIR) + File.separator + "log_herestTraining_" + iteration + ".txt"; // Reestimation complete - average log prob per frame = xxx Pattern p = Pattern.compile("^.*average log prob per frame = (.*)$"); FileReader fr = new FileReader(filename); BufferedReader reader = new BufferedReader(fr); String st = ""; Matcher m; Boolean found = false; while ((st = reader.readLine()) != null) { //System.out.println(st); m = p.matcher(st); if (m.find()) { Double logProbFrame = Double.parseDouble(m.group(1)); logProbFrame_array.add(logProbFrame); System.out.println("Average log prob per frame at iteration " + iteration + " from file is " + m.group(1) + " equal to " + logProbFrame); found = true; break; } } System.out.flush(); if (!found) { throw new RuntimeException("No match of average log prob per frame in " + filename); } //double epsilon = 0.0001; double delta; if (iteration > 1) delta = logProbFrame_array.get(iteration - 1) - logProbFrame_array.get(iteration - 2); else delta = 10000000.0; epsilon_array.add(delta); } private void insertShortPause(int i) throws Exception { String hmmDir = getProp(HTDIR) + File.separator + "hmm" + File.separator; boolean okprint = false; boolean silprint = false; System.out.println("F1:" + hmmDir + "hmm" + (i - 1) + File.separator + "hmmdefs"); System.out.println("F2:" + hmmDir + "hmm" + i + File.separator + "hmmdefs"); String line, spHmmDef = ""; // File hmmDef = new File(hmmDir+"hmm"+(i-1)+File.separator+"hmmdefs"); BufferedReader hmmDef = new BufferedReader( new FileReader(hmmDir + "hmm" + (i - 1) + File.separator + "hmmdefs")); while ((line = hmmDef.readLine()) != null) { if (line.matches("^.*\"sil\".*$")) { okprint = true; spHmmDef += "~h \"ssil\"\n"; continue; } if (okprint && line.matches("^.*ENDHMM.*$")) { spHmmDef += line + "\n"; continue; } if (okprint) { spHmmDef += line + "\n"; } } hmmDef.close(); hmmDef = new BufferedReader(new FileReader(hmmDir + "hmm" + (i - 1) + File.separator + "hmmdefs")); PrintWriter newHmmDef = new PrintWriter(new FileWriter(hmmDir + "hmm" + i + File.separator + "hmmdefs")); while ((line = hmmDef.readLine()) != null) { newHmmDef.println(line.trim()); } newHmmDef.println(spHmmDef); newHmmDef.flush(); newHmmDef.close(); hmmDef.close(); } /* * Add sp model copying the centre state of ssil */ private void insertVirtualPause(int i) throws Exception { String hmmDir = getProp(HTDIR) + File.separator + "hmm" + File.separator; boolean okprint = false; boolean okprint2 = false; boolean silprint = false; System.out.println("F1:" + hmmDir + "hmm" + (i - 1) + File.separator + "hmmdefs"); System.out.println("F2:" + hmmDir + "hmm" + i + File.separator + "hmmdefs"); String line, spHmmDef = ""; // File hmmDef = new File(hmmDir+"hmm"+(i-1)+File.separator+"hmmdefs"); BufferedReader hmmDef = new BufferedReader( new FileReader(hmmDir + "hmm" + (i - 1) + File.separator + "hmmdefs")); while ((line = hmmDef.readLine()) != null) { if (line.matches("^.*\"ssil\".*$")) { okprint = true; spHmmDef += "~h \"sp\"\n"; spHmmDef += "<BeginHMM>\n"; spHmmDef += "<NumStates> 3\n"; spHmmDef += "<State> 2\n"; continue; } //TODO: add if (okprint && line.matches("^.*<STATE> 3.*$")) { okprint2 = true; continue; } if (okprint && okprint2 & line.matches("^.*<STATE> 4.*$")) { okprint = false; okprint2 = false; continue; } if (okprint && okprint2) { spHmmDef += line + "\n"; } } spHmmDef += "<TRANSP> 3\n"; spHmmDef += "0. 1. 0.\n"; spHmmDef += "0. 0.9 0.1\n"; spHmmDef += "0. 0. 0. \n"; spHmmDef += "<ENDHMM>\n"; hmmDef.close(); hmmDef = new BufferedReader(new FileReader(hmmDir + "hmm" + (i - 1) + File.separator + "hmmdefs")); PrintWriter newHmmDef = new PrintWriter(new FileWriter(hmmDir + "hmm" + i + File.separator + "hmmdefs")); while ((line = hmmDef.readLine()) != null) { newHmmDef.println(line.trim()); } newHmmDef.println(spHmmDef); newHmmDef.flush(); newHmmDef.close(); hmmDef.close(); } /* * Add sp model copying the 3 states of ssil * remember to use appropriate AT and TI */ private void insertVirtualPauseThreeStates(int i) throws Exception { String hmmDir = getProp(HTDIR) + File.separator + "hmm" + File.separator; boolean okprint = false; boolean okprint2 = false; boolean silprint = false; System.out.println("F1:" + hmmDir + "hmm" + (i - 1) + File.separator + "hmmdefs"); System.out.println("F2:" + hmmDir + "hmm" + i + File.separator + "hmmdefs"); String line, spHmmDef = ""; // File hmmDef = new File(hmmDir+"hmm"+(i-1)+File.separator+"hmmdefs"); BufferedReader hmmDef = new BufferedReader( new FileReader(hmmDir + "hmm" + (i - 1) + File.separator + "hmmdefs")); while ((line = hmmDef.readLine()) != null) { if (line.matches("^.*\"ssil\".*$")) { okprint = true; spHmmDef += "~h \"sp\"\n"; spHmmDef += "<BeginHMM>\n"; spHmmDef += "<NumStates> 5\n"; spHmmDef += "<State> 2\n"; continue; } //TODO: add if (okprint && line.matches("^.*<STATE> 2.*$")) { okprint2 = true; continue; } if (okprint && okprint2 & line.matches("^.*<ENDHMM>.*$")) { okprint = false; okprint2 = false; continue; } if (okprint && okprint2) { spHmmDef += line + "\n"; } } /*spHmmDef += "<TRANSP> 3\n"; spHmmDef += "0. 1. 0.\n"; spHmmDef += "0. 0.9 0.1\n"; spHmmDef += "0. 0. 0. \n"; spHmmDef += "<ENDHMM>\n";*/ spHmmDef += "<ENDHMM>\n"; hmmDef.close(); hmmDef = new BufferedReader(new FileReader(hmmDir + "hmm" + (i - 1) + File.separator + "hmmdefs")); PrintWriter newHmmDef = new PrintWriter(new FileWriter(hmmDir + "hmm" + i + File.separator + "hmmdefs")); while ((line = hmmDef.readLine()) != null) { newHmmDef.println(line.trim()); } newHmmDef.println(spHmmDef); newHmmDef.flush(); newHmmDef.close(); hmmDef.close(); } /** * Force Align database for Automatic labels * @throws Exception */ private void hviteMultiplePronunciationAligning(String hmmNumber, String alignedMlf, boolean labOutput, String labDir, boolean full, String logfile, boolean cmp) throws Exception { String hvite = getProp(HTKDIR) + File.separator + "HVite"; // -A -D -V -T 1 "; // to add -A -D -V -T 1 in every function File htkFile = new File(hvite); if (!htkFile.exists()) { throw new RuntimeException("File " + htkFile.getAbsolutePath() + " does not exist"); } String configFile = getProp(HTDIR) + File.separator + "config" + File.separator + "htkTrain.conf"; String listFile = getProp(HTDIR) + File.separator + "etc" + File.separator + "htkTrain.list"; // Virtual sp change_ phoneList should be a member? // Without sp: /*String phoneList = getProp(HTDIR)+File.separator +"etc"+File.separator+"htk.phone2.list";*/ // Whit sp: String phoneList = getProp(HTDIR) + File.separator + "etc" + File.separator + "htk.phone3.list"; String hmmDef = getProp(HTDIR) + File.separator + "hmm" + File.separator + hmmNumber + File.separator + "hmmdefs"; String macros = getProp(HTDIR) + File.separator + "hmm" + File.separator + hmmNumber + File.separator + "macros"; // Virtual sp change_ phoneMlf should be a member? // Without sp: /*String phoneMlf = getProp(HTDIR)+File.separator +"etc"+File.separator+"htk.phones2.mlf";*/ // Whit sp: String phoneMlf = getProp(HTDIR) + File.separator + "etc" + File.separator + "htk.words3.mlf"; String phoneDict = getProp(HTDIR) + File.separator + "etc" + File.separator + "htk.words.dict"; Runtime rtime = Runtime.getRuntime(); //get a shell Process process = rtime.exec("/bin/bash"); //get an output stream to write to the shell //when no sp use (-m)! String cmd; String alignout, mOptioon, oOption; if (labOutput) alignout = ""; else alignout = " -i " + alignedMlf; if (full) { if (labOutput) mOptioon = ""; else mOptioon = " -m"; cmd = "( cd " + getProp(HTDIR) + "; " + hvite + " " + HTK_SO + " -b sil -l " + labDir + " -C " + configFile + mOptioon + " -a -H " + macros + " -H " + hmmDef + alignout + " -t 250.0 -y lab" + " -I " + phoneMlf + " -S " + listFile + " " + phoneDict + " " + phoneList + " > " + logfile + "; exit )\n"; } else { if (cmp) oOption = " -o TS"; else oOption = " -o W"; cmd = "( cd " + getProp(HTDIR) + "; " + hvite + " " + HTK_SO + " -b sil -l " + labDir + oOption + " -C " + configFile + " -m -a -H " + macros + " -H " + hmmDef + alignout + " -t 250.0 -y lab" + " -I " + phoneMlf + " -S " + listFile + " " + phoneDict + " " + phoneList + " > " + logfile + "; exit )\n"; } PrintWriter pw = new PrintWriter(new OutputStreamWriter(process.getOutputStream())); System.out.println(cmd); pw.println(cmd); pw.flush(); //shut down pw.close(); process.waitFor(); // check exit value if (process.exitValue() != 0) { BufferedReader errorReader = new BufferedReader(new InputStreamReader(process.getErrorStream())); throw new MaryConfigurationException(errorReader.readLine()); } } /** * Force Align database for Automatic labels * @throws Exception */ private void hviteAligning() throws Exception { String hvite = getProp(HTKDIR) + File.separator + "HVite"; // -A -D -V -T 1 "; // to add -A -D -V -T 1 in every function File htkFile = new File(hvite); if (!htkFile.exists()) { throw new RuntimeException("File " + htkFile.getAbsolutePath() + " does not exist"); } String configFile = getProp(HTDIR) + File.separator + "config" + File.separator + "htkTrain.conf"; String listFile = getProp(HTDIR) + File.separator + "etc" + File.separator + "htkTrain.list"; // Virtual sp change_ phoneList should be a member? // Without sp: /*String phoneList = getProp(HTDIR)+File.separator +"etc"+File.separator+"htk.phone2.list";*/ // Whit sp: String phoneList = getProp(HTDIR) + File.separator + "etc" + File.separator + "htk.phone3.list"; String hmmDef = getProp(HTDIR) + File.separator + "hmm" + File.separator + "hmm-final" + File.separator + "hmmdefs"; String macros = getProp(HTDIR) + File.separator + "hmm" + File.separator + "hmm-final" + File.separator + "macros"; // Virtual sp change_ phoneMlf should be a member? // Without sp: /*String phoneMlf = getProp(HTDIR)+File.separator +"etc"+File.separator+"htk.phones2.mlf";*/ // Whit sp: String phoneMlf = getProp(HTDIR) + File.separator + "etc" + File.separator + "htk.phones3.mlf"; String alignedMlf = getProp(HTDIR) + File.separator + "aligned.mlf"; String phoneDict = getProp(HTDIR) + File.separator + "etc" + File.separator + "htk.phone.dict"; String labDir = getProp(HTDIR) + File.separator + "lab"; Runtime rtime = Runtime.getRuntime(); //get a shell Process process = rtime.exec("/bin/bash"); //get an output stream to write to the shell //when no sp use (-m)! PrintWriter pw = new PrintWriter(new OutputStreamWriter(process.getOutputStream())); System.out.println("( cd " + getProp(HTDIR) + "; " + hvite + " " + HTK_SO + " -b sil -l " + labDir + " -o W -C " + configFile + " -a -H " + macros + " -H " + hmmDef + " -i " + alignedMlf + " -t 250.0 -y lab" + " -I " + phoneMlf + " -S " + listFile + " " + phoneDict + " " + phoneList + " > log_hviteAligning.txt" + "; exit )\n"); pw.println("( cd " + getProp(HTDIR) + "; " + hvite + " " + HTK_SO + " -b sil -l " + labDir + " -o W -C " + configFile + " -a -H " + macros + " -H " + hmmDef + " -i " + alignedMlf + " -t 250.0 -y lab" + " -I " + phoneMlf + " -S " + listFile + " " + phoneDict + " " + phoneList + " > log_hviteAligning.txt" + "; exit )\n"); pw.flush(); //shut down pw.close(); process.waitFor(); // check exit value if (process.exitValue() != 0) { BufferedReader errorReader = new BufferedReader(new InputStreamReader(process.getErrorStream())); throw new MaryConfigurationException(errorReader.readLine()); } } private void htkExtraModels() throws Exception { String hlstats = getProp(HTKDIR) + File.separator + "HLStats"; String hbuild = getProp(HTKDIR) + File.separator + "HBuild"; File htkFile = new File(hlstats); if (!htkFile.exists()) { throw new RuntimeException("File " + htkFile.getAbsolutePath() + " does not exist"); } String configFile = getProp(HTDIR) + File.separator + "config" + File.separator + "htkTrain.conf"; String bigFile = getProp(HTDIR) + File.separator + "etc" + File.separator + "htk.phones.big"; String phoneList = getProp(HTDIR) + File.separator + "etc" + File.separator + "htk.phone.list"; String phoneMlf = getProp(HTDIR) + File.separator + "etc" + File.separator + "htk.phones.mlf"; String phoneDict = getProp(HTDIR) + File.separator + "etc" + File.separator + "htk.phone.dict"; String phoneAugDict = getProp(HTDIR) + File.separator + "etc" + File.separator + "htk.aug.phone.dict"; String phoneAugList = getProp(HTDIR) + File.separator + "etc" + File.separator + "htk.aug.phone.list"; String netFile = getProp(HTDIR) + File.separator + "etc" + File.separator + "htk.phones.net"; Runtime rtime = Runtime.getRuntime(); //get a shell Process process = rtime.exec("/bin/bash"); //get an output stream to write to the shell PrintWriter pw = new PrintWriter(new OutputStreamWriter(process.getOutputStream())); System.out.println("( cd " + getProp(HTDIR) + "; " + hlstats + " -T 1 -C " + configFile + " -b " + bigFile + " -o " + phoneList + " " + phoneMlf + " > log_hlstats.txt" + "; exit )\n"); pw.println("( cd " + getProp(HTDIR) + "; " + hlstats + " -T 1 -C " + configFile + " -b " + bigFile + " -o " + phoneList + " " + phoneMlf + " > log_hlstats.txt" + "; exit )\n"); pw.flush(); //shut down pw.close(); process.waitFor(); // check exit value if (process.exitValue() != 0) { BufferedReader errorReader = new BufferedReader(new InputStreamReader(process.getErrorStream())); throw new MaryConfigurationException(errorReader.readLine()); } String fileDict = FileUtils.getFileAsString(new File(phoneDict), "ASCII"); PrintWriter augPhoneDict = new PrintWriter(new FileWriter(phoneAugDict)); augPhoneDict.println("!ENTER sil"); augPhoneDict.print(fileDict); augPhoneDict.println("!EXIT sil"); augPhoneDict.flush(); augPhoneDict.close(); String fileList = FileUtils.getFileAsString(new File(phoneList), "ASCII"); PrintWriter augPhoneList = new PrintWriter(new FileWriter(phoneAugList)); augPhoneList.println("!ENTER"); augPhoneList.print(fileList); augPhoneList.println("!EXIT"); augPhoneList.flush(); augPhoneList.close(); rtime = Runtime.getRuntime(); //get a shell process = rtime.exec("/bin/bash"); //get an output stream to write to the shell pw = new PrintWriter(new OutputStreamWriter(process.getOutputStream())); System.out.println("( cd " + getProp(HTDIR) + "; " + hbuild + " -T 1 -C " + configFile + " -n " + bigFile + " " + phoneAugList + " " + netFile + " > log_hbuild.txt" + "; exit )\n"); pw.println("( cd " + getProp(HTDIR) + "; " + hbuild + " -T 1 -C " + configFile + " -n " + bigFile + " " + phoneAugList + " " + netFile + " > log_hbuild.txt" + "; exit )\n"); pw.flush(); //shut down pw.close(); process.waitFor(); // check exit value if (process.exitValue() != 0) { BufferedReader errorReader = new BufferedReader(new InputStreamReader(process.getErrorStream())); throw new MaryConfigurationException(errorReader.readLine()); } } /** * Create phone sequence file, which is * used for Alignment * @throws Exception */ private void getPhoneSequence() throws Exception { // open transcription file used for labeling PrintWriter transLabelOut = new PrintWriter( new FileOutputStream(new File(outputDir + "/" + "htk.phones.mlf"))); PrintWriter transLabelOut1 = new PrintWriter( new FileOutputStream(new File(outputDir + "/" + "htk.phones2.mlf"))); PrintWriter transLabelOut2 = new PrintWriter( new FileOutputStream(new File(outputDir + "/" + "htk.phones3.mlf"))); // words and phones (used to compare the results with multiple pronunctiaons) PrintWriter transLabelOut2b = new PrintWriter( new FileOutputStream(new File(outputDir + "/" + "htk.words_phones3.mlf"))); // words used for multiple pronunciations forced alignement PrintWriter transLabelOut3 = new PrintWriter( new FileOutputStream(new File(outputDir + "/" + "htk.words3.mlf"))); String phoneSeq; String wordSeq; transLabelOut.println("#!MLF!#"); transLabelOut1.println("#!MLF!#"); transLabelOut2.println("#!MLF!#"); transLabelOut2b.println("#!MLF!#"); transLabelOut3.println("#!MLF!#"); for (int i = 0; i < bnl.getLength(); i++) { transLabelOut.println("\"*/" + bnl.getName(i) + labExt + "\""); transLabelOut1.println("\"*/" + bnl.getName(i) + labExt + "\""); transLabelOut2.println("\"*/" + bnl.getName(i) + labExt + "\""); transLabelOut2b.println("\"*/" + bnl.getName(i) + labExt + "\""); transLabelOut3.println("\"*/" + bnl.getName(i) + labExt + "\""); //phoneSeq = getSingleLine(bnl.getName(i)); phoneSeq = getLineFromXML(bnl.getName(i), false, false); transLabelOut.println(phoneSeq.trim()); phoneSeq = getLineFromXML(bnl.getName(i), true, false); transLabelOut1.println(phoneSeq.trim()); phoneSeq = getLineFromXML(bnl.getName(i), true, true); transLabelOut2.println(phoneSeq.trim()); // word phoneSeq = getWordandPhoneLineFromXML(bnl.getName(i), true, true); transLabelOut2b.println(phoneSeq.trim()); wordSeq = getWordLineFromXML(bnl.getName(i), true, true); transLabelOut3.println(wordSeq.trim()); //System.out.println( " " + bnl.getName(i) ); } transLabelOut.flush(); transLabelOut.close(); transLabelOut1.flush(); transLabelOut1.close(); transLabelOut2.flush(); transLabelOut2.close(); transLabelOut2b.flush(); transLabelOut2b.close(); transLabelOut3.flush(); transLabelOut3.close(); } /** * Get phone sequence from a single feature file * @param basename * @return String * @throws Exception */ private String getLineFromXML(String basename, boolean spause, boolean vpause) throws Exception { String line; String phoneSeq; Matcher matcher; Pattern pattern; StringBuilder alignBuff = new StringBuilder(); //alignBuff.append(basename); DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); DocumentBuilder builder = factory.newDocumentBuilder(); Document doc = builder.parse(new File(getProp(PROMPTALLOPHONESDIR) + "/" + basename + xmlExt)); XPath xpath = XPathFactory.newInstance().newXPath(); NodeList tokens = (NodeList) xpath.evaluate("//t | //boundary", doc, XPathConstants.NODESET); alignBuff.append(collectTranscription(tokens)); phoneSeq = alignBuff.toString(); pattern = Pattern.compile("pau ssil "); matcher = pattern.matcher(phoneSeq); phoneSeq = matcher.replaceAll("sil "); pattern = Pattern.compile(" ssil pau$"); matcher = pattern.matcher(phoneSeq); phoneSeq = matcher.replaceAll(" sil"); if (!vpause) { /* TODO: Extra code need to write * to maintain minimum number of short sil. * or consider word boundaries as ssil. */ /* * virtual silence on word boundaries * are matched in sp */ pattern = Pattern.compile("vssil"); matcher = pattern.matcher(phoneSeq); phoneSeq = matcher.replaceAll(""); } else { /* * virtual silence on word boundaries * are matched in sp */ pattern = Pattern.compile("vssil"); matcher = pattern.matcher(phoneSeq); phoneSeq = matcher.replaceAll("sp"); } // checking if (!spause) { pattern = Pattern.compile("ssil"); matcher = pattern.matcher(phoneSeq); phoneSeq = matcher.replaceAll(""); } phoneSeq += " ."; pattern = Pattern.compile("\\s+"); matcher = pattern.matcher(phoneSeq); phoneSeq = matcher.replaceAll("\n"); //System.out.println(phoneSeq); return phoneSeq; } /** * Get word and phone sequence from a single feature file * @param basename * @return String * @throws Exception */ private String getWordandPhoneLineFromXML(String basename, boolean spause, boolean vpause) throws Exception { String line; String phoneSeq; Matcher matcher; Pattern pattern; StringBuilder alignBuff = new StringBuilder(); //alignBuff.append(basename); DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); DocumentBuilder builder = factory.newDocumentBuilder(); Document doc = builder.parse(new File(getProp(PROMPTALLOPHONESDIR) + "/" + basename + xmlExt)); XPath xpath = XPathFactory.newInstance().newXPath(); NodeList tokens = (NodeList) xpath.evaluate("//t | //boundary", doc, XPathConstants.NODESET); alignBuff.append(collectTranscriptionAndWord(tokens)); phoneSeq = alignBuff.toString(); pattern = Pattern.compile("pau ssil "); matcher = pattern.matcher(phoneSeq); phoneSeq = matcher.replaceAll("sil "); pattern = Pattern.compile(" ssil pau$"); matcher = pattern.matcher(phoneSeq); phoneSeq = matcher.replaceAll(" sil"); if (!vpause) { /* TODO: Extra code need to write * to maintain minimum number of short sil. * or consider word boundaries as ssil. */ /* * virtual silence on word boundaries * are matched in sp */ pattern = Pattern.compile("vssil"); matcher = pattern.matcher(phoneSeq); phoneSeq = matcher.replaceAll(""); } else { /* * virtual silence on word boundaries * are matched in sp */ pattern = Pattern.compile("vssil"); matcher = pattern.matcher(phoneSeq); phoneSeq = matcher.replaceAll("sp"); } // checking if (!spause) { pattern = Pattern.compile("ssil"); matcher = pattern.matcher(phoneSeq); phoneSeq = matcher.replaceAll(""); } phoneSeq += " ."; pattern = Pattern.compile("\\s+"); matcher = pattern.matcher(phoneSeq); phoneSeq = matcher.replaceAll("\n"); pattern = Pattern.compile("-"); matcher = pattern.matcher(phoneSeq); phoneSeq = matcher.replaceAll(" "); //System.out.println(phoneSeq); return phoneSeq; } /** * Get word sequence from a single feature file * @param basename * @return String * @throws Exception */ private String getWordLineFromXML(String basename, boolean spause, boolean vpause) throws Exception { String line; String phoneSeq; Matcher matcher; Pattern pattern; StringBuilder alignBuff = new StringBuilder(); //alignBuff.append(basename); DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); DocumentBuilder builder = factory.newDocumentBuilder(); Document doc = builder.parse(new File(getProp(PROMPTALLOPHONESDIR) + "/" + basename + xmlExt)); XPath xpath = XPathFactory.newInstance().newXPath(); NodeList tokens = (NodeList) xpath.evaluate("//t | //boundary", doc, XPathConstants.NODESET); alignBuff.append(collectWordTranscription(tokens)); phoneSeq = alignBuff.toString(); pattern = Pattern.compile("pau ssil "); matcher = pattern.matcher(phoneSeq); phoneSeq = matcher.replaceAll("sil "); pattern = Pattern.compile(" ssil pau$"); matcher = pattern.matcher(phoneSeq); phoneSeq = matcher.replaceAll(" sil"); if (!vpause) { /* TODO: Extra code need to write * to maintain minimum number of short sil. * or consider word boundaries as ssil. */ /* * virtual silence on word boundaries * are matched in sp */ pattern = Pattern.compile("vssil"); matcher = pattern.matcher(phoneSeq); phoneSeq = matcher.replaceAll(""); } else { /* * virtual silence on word boundaries * are matched in sp */ pattern = Pattern.compile("vssil"); matcher = pattern.matcher(phoneSeq); phoneSeq = matcher.replaceAll(""); } // checking if (!spause) { pattern = Pattern.compile("ssil"); matcher = pattern.matcher(phoneSeq); phoneSeq = matcher.replaceAll(""); } phoneSeq += " ."; pattern = Pattern.compile("\\s+"); matcher = pattern.matcher(phoneSeq); phoneSeq = matcher.replaceAll("\n"); //System.out.println(phoneSeq); return phoneSeq; } /** * * This computes a string of phonetic symbols out of an prompt allophones mary xml: * - standard phones are taken from "ph" attribute * @param tokens * @return */ private String collectTranscription(NodeList tokens) { // TODO: make delims argument // String Tokenizer devides transcriptions into syllables // syllable delimiters and stress symbols are retained String delims = "',-"; // String storing the original transcription begins with a pause String orig = " pau "; // get original phone String for (int tNr = 0; tNr < tokens.getLength(); tNr++) { Element token = (Element) tokens.item(tNr); // only look at it if there is a sampa to change if (token.hasAttribute("ph")) { String sampa = token.getAttribute("ph"); List<String> sylsAndDelims = new ArrayList<String>(); StringTokenizer sTok = new StringTokenizer(sampa, delims, true); while (sTok.hasMoreElements()) { String currTok = sTok.nextToken(); if (delims.indexOf(currTok) == -1) { // current Token is no delimiter for (Allophone ph : allophoneSet.splitIntoAllophones(currTok)) { // orig += ph.name() + " "; if (ph.name().trim().equals("_")) continue; orig += replaceTrickyPhones(ph.name().trim()) + " "; } // ... for each phone } // ... if no delimiter } // ... while there are more tokens } // TODO: simplify if (token.getTagName().equals("t")) { // if the following element is no boundary, insert a non-pause delimiter if (tNr == tokens.getLength() - 1 || !((Element) tokens.item(tNr + 1)).getTagName().equals("boundary")) { orig += "vssil "; // word boundary } } else if (token.getTagName().equals("boundary")) { orig += "ssil "; // phrase boundary } else { // should be "t" or "boundary" elements assert (false); } } // ... for each t-Element orig += "pau"; return orig; } // word and phones transcriptions private String collectTranscriptionAndWord(NodeList tokens) { // TODO: make delims argument // String Tokenizer devides transcriptions into syllables // syllable delimiters and stress symbols are retained String delims = "',-"; // String storing the original transcription begins with a pause String orig = " pau "; String word, HTKWORD; boolean first_word_phone = true; // get original phone String for (int tNr = 0; tNr < tokens.getLength(); tNr++) { Element token = (Element) tokens.item(tNr); // only look at it if there is a sampa to change if (token.hasAttribute("ph")) { word = token.getTextContent().trim(); HTKWORD = word.toUpperCase(); first_word_phone = true; String sampa = token.getAttribute("ph"); List<String> sylsAndDelims = new ArrayList<String>(); StringTokenizer sTok = new StringTokenizer(sampa, delims, true); while (sTok.hasMoreElements()) { String currTok = sTok.nextToken(); if (delims.indexOf(currTok) == -1) { // current Token is no delimiter for (Allophone ph : allophoneSet.splitIntoAllophones(currTok)) { // orig += ph.name() + " "; if (ph.name().trim().equals("_")) continue; orig += replaceTrickyPhones(ph.name().trim()); if (first_word_phone) { orig += "-" + HTKWORD + " "; first_word_phone = false; } else orig += " "; } // ... for each phone } // ... if no delimiter } // ... while there are more tokens } // TODO: simplify if (token.getTagName().equals("t")) { // if the following element is no boundary, insert a non-pause delimiter if (tNr == tokens.getLength() - 1 || !((Element) tokens.item(tNr + 1)).getTagName().equals("boundary")) { orig += "vssil "; // word boundary } } else if (token.getTagName().equals("boundary")) { orig += "ssil "; // phrase boundary } else { // should be "t" or "boundary" elements assert (false); } } // ... for each t-Element orig += "pau"; return orig; } /** * * This computes a string of words out of an prompt allophones mary xml: * - standard phones are taken from "ph" attribute * @param tokens * @return */ private String collectWordTranscription(NodeList tokens) { // TODO: make delims argument // String Tokenizer devides transcriptions into syllables // syllable delimiters and stress symbols are retained String delims = "',-"; // String storing the original transcription begins with a pause String orig = " pau "; String HTKWORD_xml_transcription; String mary_transcription; String HTKWORD, word; // get original phone String for (int tNr = 0; tNr < tokens.getLength(); tNr++) { Element token = (Element) tokens.item(tNr); // only look at it if there is a sampa to change if (token.hasAttribute("ph")) { HTKWORD_xml_transcription = ""; mary_transcription = ""; String sampa = token.getAttribute("ph"); mary_transcription = sampa.trim().replace(" ", ""); List<String> sylsAndDelims = new ArrayList<String>(); StringTokenizer sTok = new StringTokenizer(sampa, delims, true); while (sTok.hasMoreElements()) { String currTok = sTok.nextToken(); if (delims.indexOf(currTok) == -1) { // current Token is no delimiter for (Allophone ph : allophoneSet.splitIntoAllophones(currTok)) { // orig += ph.name() + " "; if (ph.name().trim().equals("_")) continue; HTKWORD_xml_transcription += replaceTrickyPhones(ph.name().trim()) + " "; //globalwordlexicon += HTKWORD + " " + HTKWORD_xml_transcription; } // ... for each phone } // ... if no delimiter } // ... while there are more tokens word = token.getTextContent().trim(); HTKWORD = word.toUpperCase(); HTKWORD_xml_transcription = HTKWORD_xml_transcription.trim(); if ((token.hasAttribute("g2p_method") && token.getAttribute("g2p_method").equals("privatedict")) // this is for rawxml entry with token with ph attribute || !token.hasAttribute("g2p_method")) { HTKWORD = HTKWORD + "_" + HTKWORD_xml_transcription.replaceAll(" ", ""); //System.out.println("HTKWORD private lexicon or rawxml ph: " + HTKWORD); } // dictionary //System.out.println("HTKWORD: " + HTKWORD + " HTKWORD_xml_transcription: " + HTKWORD_xml_transcription); HTKdictionary.add(HTKWORD + " " + HTKWORD_xml_transcription); Totaldictionary .add(HTKWORD + " " + HTKWORD_xml_transcription.replace(" ", "") + " " + mary_transcription); String[] entries; entries = lexicon.lookup(word); //insert here all the different possible transcriptions for (int i = 0; i < entries.length; i++) { String HTKTranscription = entries[i]; mary_transcription = HTKTranscription.replace(" ", ""); HTKTranscription = HTKTranscription.replace("' ", ""); HTKTranscription = HTKTranscription.replace("- ", ""); //TODO: replaceTrickyPhones HTKTranscription HTKdictionary.add(HTKWORD + " " + HTKTranscription); Totaldictionary .add(HTKWORD + " " + HTKTranscription.replace(" ", "") + " " + mary_transcription); } orig += HTKWORD + " "; } // TODO: simplify if (token.getTagName().equals("t")) { // if the following element is no boundary, insert a non-pause delimiter if (tNr == tokens.getLength() - 1 || !((Element) tokens.item(tNr + 1)).getTagName().equals("boundary")) { orig += "vssil "; // word boundary } } else if (token.getTagName().equals("boundary")) { orig += "ssil "; // phrase boundary } else { // should be "t" or "boundary" elements assert (false); } } // ... for each t-Element orig += "pau"; return orig; } /** * Post Processing single Label file * and write on OUTLABDIR * @param basename * @throws Exception */ private void convertSingleLabelFile(String basename) throws Exception { String line; String previous, current; String regexp = "\\spau|\\sssil"; //Compile regular expression Pattern pattern = Pattern.compile(regexp); File labDir = new File(getProp(OUTLABDIR)); if (!labDir.exists()) { labDir.mkdir(); } File labelFile = new File(getProp(HTDIR) + File.separator + "tmplab" + File.separator + basename + labExt); if (!labelFile.exists()) { System.err.println("WARNING: " + basename + " label file not created with HTK."); return; } BufferedReader labelIn = new BufferedReader(new InputStreamReader(new FileInputStream(labelFile))); PrintWriter labelOut = new PrintWriter(new FileOutputStream(new File(labDir + "/" + basename + labExt))); previous = labelIn.readLine(); while ((line = labelIn.readLine()) != null) { //Replace all occurrences of pattern in input Matcher matcher = pattern.matcher(line); current = matcher.replaceAll(" _"); if (previous.endsWith("_") && current.endsWith("_")) { previous = current; continue; } labelOut.println(previous); previous = current; } labelOut.println(previous); labelOut.flush(); labelOut.close(); labelIn.close(); } /** * To convert HTK Label format to MARY lab format * @throws Exception */ private void getProperLabelFormat() throws Exception { String alignedMlf = getProp(HTDIR) + File.separator + "aligned.mlf"; BufferedReader htkLab = new BufferedReader(new FileReader(alignedMlf)); //File labDir = new File(getProp(OUTLABDIR)); File labDir = new File(getProp(HTDIR) + File.separator + "tmplab"); if (!labDir.exists()) labDir.mkdir(); String header = htkLab.readLine().trim(); if (!header.equals("#!MLF!#")) { System.err.println("Header format not supported"); throw new RuntimeException("Header format not supported"); } String line; while ((line = htkLab.readLine()) != null) { line = line.trim(); String MLFfileName = line.substring(1, line.length() - 1); String Basename = new File(MLFfileName).getName(); System.out.println("Basename: " + Basename); String fileName = labDir.getCanonicalPath() + File.separator + Basename; //line.replaceAll("\"", ""); //System.err.println("LINE: "+fileName); PrintWriter pw = new PrintWriter(new FileWriter(fileName)); pw.println("#"); while (true) { String nline = htkLab.readLine().trim(); if (nline.equals(".")) break; StringTokenizer st = new StringTokenizer(nline); Double tStart = Double.parseDouble(st.nextToken().trim()); Double tStamp = Double.parseDouble(st.nextToken().trim()); String phoneSeg = replaceBackTrickyPhones(st.nextToken().trim()); //System.out.println( "start " + tStart + " stop " + tStamp + " " + phoneSeg); Double dur = tStamp - tStart; Double durms = dur / 10000; if (phoneSeg.equals("sp")) { if (dur == 0) { //System.out.println("sp to delete!!!"); continue; } /*else if (dur <= 150000) //150000 = 15 ms { //TODO: A better post processing should be done: i.e. check the previous and the next phone ... System.out.println("sp <= 15 ms to delete!!!"); continue; }*/ else { System.out.println(fileName + ": a sp (virtual) pause with duration: " + durms + " ms, has been detected at " + tStart + " " + tStamp); /* * The following gawk lines can be used to inspect very long sp pause: * gawk 'match($0, /^(.*): a sp.*duration: ([0-9]+\.[0-9]+) ms.*$/, arr) {if (arr[2]>200) {print "file:" arr[1] " duration:" arr[2]} }' nohup.out * gawk 'match($0, /^(.*): a sp.*duration: ([0-9]+\.[0-9]+) ms.*$/, arr) {if (arr[2]>400) {print $0} }' nohup.out */ } } else if (phoneSeg.equals("ssil")) { if (dur == 0) { System.out.println(fileName + ": ssil to delete!!! a ssil (pause associated with punctuation) with duration: " + durms + " ms, has been detected at " + tStart + " " + tStamp); continue; } else { System.out.println(fileName + ": a ssil (pause associated with punctuation) with duration: " + durms + " ms, has been detected at " + tStart + " " + tStamp); /* * The following gawk lines can be used to inspect very short ssil pause: * gawk 'match($0, /^(.*): a ssil.*duration: ([0-9]+\.[0-9]+) ms.*$/, arr) {if (arr[2]>200) {print "file:" arr[1] " duration:" arr[2]} }' nohup.out * gawk 'match($0, /^(.*): a ssil.*duration: ([0-9]+\.[0-9]+) ms.*$/, arr) {if (arr[2]>400) {print $0} }' nohup.out */ } } if (phoneSeg.equals("sil") || phoneSeg.equals("ssil") || phoneSeg.equals("sp")) phoneSeg = "_"; pw.println(tStamp / 10000000 + " 125 " + phoneSeg); } pw.flush(); pw.close(); } for (int i = 0; i < bnl.getLength(); i++) { convertSingleLabelFile(bnl.getName(i)); //System.out.println( " " + bnl.getName(i) ); } } /** * Converting text to RAWMARYXML with Locale * @param locale * @return */ public static String getMaryXMLHeaderWithInitialBoundary(String locale) { return "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n" + "<maryxml version=\"0.4\"\n" + "xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\"\n" + "xmlns=\"http://mary.dfki.de/2002/MaryXML\"\n" + "xml:lang=\"" + locale + "\">\n" + "<boundary duration=\"100\"/>\n"; } /** * Provide the progress of computation, in percent, or -1 if * that feature is not implemented. * @return -1 if not implemented, or an integer between 0 and 100. */ public int getProgress() { return progress; } /** Translation table for labels which are incompatible with HTK or shell filenames * See common_routines.pl in HTS training. * @param lab * @return String */ public String replaceTrickyPhones(String lab) { String s = lab; /** the replace is done for the labels: phone, prev_phone and next_phone */ /** DE (replacements in German phone set) */ if (lab.contentEquals("6")) s = "ER6"; else if (lab.contentEquals("=6")) s = "ER66"; else if (lab.contentEquals("2:")) s = "EU22"; else if (lab.contentEquals("2")) s = "EU2"; else if (lab.contentEquals("9")) s = "EU9"; else if (lab.contentEquals("9~")) s = "UM9"; else if (lab.contentEquals("e~")) s = "IMe"; else if (lab.contentEquals("a~")) s = "ANa"; else if (lab.contentEquals("o~")) s = "ONo"; else if (lab.contentEquals("?")) s = "gstop"; /** EN (replacements in English phone set) */ //else if (lab.contentEquals("r=") ) // s = "rr"; return s; } /** Translation table for labels which are incompatible with HTK or shell filenames * See common_routines.pl in HTS training. * In this function the phones as used internally in HTSEngine are changed * back to the Mary TTS set, this function is necessary when correcting the * actual durations of AcousticPhonemes. * @param lab * @return String */ public String replaceBackTrickyPhones(String lab) { String s = lab; /** DE (replacements in German phone set) */ if (lab.contentEquals("ER6")) s = "6"; else if (lab.contentEquals("ER66")) /* CHECK ??? */ s = "=6"; else if (lab.contentEquals("EU2")) s = "2"; else if (lab.contentEquals("EU22")) s = "2:"; else if (lab.contentEquals("EU9")) s = "9"; else if (lab.contentEquals("UM9")) s = "9~"; else if (lab.contentEquals("IMe")) s = "e~"; else if (lab.contentEquals("ANa")) s = "a~"; else if (lab.contentEquals("ONo")) s = "o~"; else if (lab.contentEquals("gstop")) s = "?"; /** EN (replacements in English phone set) */ //else if (lab.contentEquals("rr") ) // s = "r="; //System.out.println("LAB=" + s); return s; } }