List of usage examples for android.media MediaCodec queueInputBuffer
public final void queueInputBuffer(int index, int offset, int size, long presentationTimeUs, int flags) throws CryptoException
From source file:com.musicplayer.AudioDecoderThread.java
/** * After decoding AAC, Play using Audio Track. * // w ww. ja va2 s .com */ public void processTrack(Uri syncContentUri, final Genre classLabel, Context context, ProcessTrackRunnable lock) { // INITIALISE EXTRACTOR AND DECODER Log.v("", "Break Point 1"); MediaExtractor extractor = new MediaExtractor(); int sampleRate = 0; Uri contentUri = null; synchronized (lock) { contentUri = syncContentUri; } try { extractor.setDataSource(context, contentUri, null); } catch (IOException e) { e.printStackTrace(); } int channel = 0; for (int i = 0; i < extractor.getTrackCount(); i++) { MediaFormat format = extractor.getTrackFormat(i); String mime = format.getString(MediaFormat.KEY_MIME); if (mime.startsWith("audio/")) { extractor.selectTrack(i); Log.d("", "format : " + format); // ByteBuffer csd = format.getByteBuffer("csd-0"); // if(csd == null){ // Log.v("", "csd is null"); // } else{ // Log.v("", "csd is not null"); // } // for (int k = 0; k < csd.capacity(); ++k) { // Log.v("", "inside for loop 1"); // Log.e("TAG", "csd : " + csd.array()[k]); // } sampleRate = format.getInteger(MediaFormat.KEY_SAMPLE_RATE); channel = format.getInteger(MediaFormat.KEY_CHANNEL_COUNT); break; } } // MediaFormat format = makeAACCodecSpecificData(MediaCodecInfo.CodecProfileLevel.AACObjectLC, mSampleRate, channel); // if (format == null) // return; int countt = 0; boolean found = false; MediaFormat format = null; String mime = null; while (countt < extractor.getTrackCount() && !found) { format = extractor.getTrackFormat(countt); mime = format.getString(MediaFormat.KEY_MIME); sampleRate = format.getInteger(MediaFormat.KEY_SAMPLE_RATE); if (mime.startsWith("audio/")) { found = true; } countt++; } //format = mExtractor.getTrackFormat(count); //MediaCodecInfo codec = selectCodec(mime); //String name = codec.getName(); MediaCodec decoder = MediaCodec.createDecoderByType(mime); //mDecoder = MediaCodec.createDecoderByType("audio/mp4a-latm"); decoder.configure(format, null, null, 0); if (decoder == null) { Log.e("DecodeActivity", "Can't find video info!"); return; } decoder.start(); Log.v("", "Break Point 2"); // Get decoded bytes ByteBuffer[] inputBuffers = decoder.getInputBuffers(); ByteBuffer[] outputBuffers = decoder.getOutputBuffers(); BufferInfo info = new BufferInfo(); // int buffsize = AudioTrack.getMinBufferSize(sampleRate, AudioFormat.CHANNEL_OUT_STEREO, AudioFormat.ENCODING_PCM_16BIT); // // create an audiotrack object // AudioTrack audioTrack = new AudioTrack(AudioManager.STREAM_MUSIC, sampleRate, // AudioFormat.CHANNEL_OUT_STEREO, // AudioFormat.ENCODING_PCM_16BIT, // buffsize, // AudioTrack.MODE_STREAM); // audioTrack.play(); extractor.seekTo(WINDOW_START, MediaExtractor.SEEK_TO_CLOSEST_SYNC); long start = SystemClock.elapsedRealtimeNanos(); Log.v("", "Break Point 3"); // MUSICAL SURFACE FEATURES double[] flux = new double[NUM_CHUNKS]; double[] zeroCrossings = new double[NUM_CHUNKS]; double[] centroid = new double[NUM_CHUNKS]; int[] rolloff = new int[NUM_CHUNKS]; double[] rolloffFreq = new double[NUM_CHUNKS]; double lowEnergy = 0.0; // Means across all chunks double fluxMean = 0.0; double zeroCrossingsMean = 0; double centroidMean = 0.0; double rolloffMean = 0; // Standard deviations across all chunks double fluxStdDeviation = 0.0; double zeroCrossingsStdDeviation = 0; double centroidStdDeviation = 0.0; double rolloffStdDeviation = 0; // Initialise some variables to use while iterating double[] fftSums = new double[NUM_CHUNKS]; int iter = 0; int count = 0; FastFourierTransformer transformer = new FastFourierTransformer(DftNormalization.STANDARD); double po2 = 0.0; Complex[] input = null; Complex[] output = null; Complex[] previousOutput = null; Complex[] temp = null; double frequency = 0.0; double centroidNum = 0.0; double centroidDen = 0.0; double fftValue = 0.0; double fftPrevious = 0.0; double fluxSquared = 0.0; int r = 0; boolean foundRolloff = false; double sum = 0; ArrayList<Double> data = new ArrayList<Double>(); ArrayList<Double> currentChunk = new ArrayList<Double>(); int gap = 0; int tempCount = 0; byte[] chunk = null; ArrayList<Double> outputExample = new ArrayList<Double>(); double normConst = 0.0; // Iterate through the chunks Log.v("", "count: " + String.valueOf(count)); while (!eosReceived && count < NUM_CHUNKS) { Log.v("", "Break Point " + String.valueOf(count + 4)); Log.v("", "Inside While Loop Break Point 1"); if (count == 0) { // Log.v("", "Timestamp of chunk 0: " + String.valueOf(extractor.getSampleTime())); } int inIndex = decoder.dequeueInputBuffer(TIMEOUT_US); if (inIndex >= 0) { ByteBuffer buffer = inputBuffers[inIndex]; int sampleSize = extractor.readSampleData(buffer, 0); if (sampleSize < 0) { // We shouldn't stop the playback at this point, just pass the EOS // flag to mDecoder, we will get it again from the // dequeueOutputBuffer //Log.d("DecodeActivity", "InputBuffer BUFFER_FLAG_END_OF_STREAM"); decoder.queueInputBuffer(inIndex, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM); } else { decoder.queueInputBuffer(inIndex, 0, sampleSize, extractor.getSampleTime(), 0); extractor.advance(); } int outIndex = decoder.dequeueOutputBuffer(info, TIMEOUT_US); Log.v("", "Inside While Loop Break Point 2"); switch (outIndex) { case MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED: Log.d("DecodeActivity", "INFO_OUTPUT_BUFFERS_CHANGED"); outputBuffers = decoder.getOutputBuffers(); break; case MediaCodec.INFO_OUTPUT_FORMAT_CHANGED: MediaFormat mediaFormat = decoder.getOutputFormat(); Log.d("DecodeActivity", "New format " + mediaFormat); // audioTrack.setPlaybackRate(mediaFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE)); break; case MediaCodec.INFO_TRY_AGAIN_LATER: Log.d("DecodeActivity", "dequeueOutputBuffer timed out!"); break; default: Log.v("", "Inside While Loop Break Point 3"); ByteBuffer outBuffer = outputBuffers[outIndex]; //Log.v("DecodeActivity", "We can't use this buffer but render it due to the API limit, " + outBuffer); chunk = new byte[info.size]; if (chunk.length == 0) { continue; } outBuffer.get(chunk); // Read the buffer all at once outBuffer.clear(); // ** MUST DO!!! OTHERWISE THE NEXT TIME YOU GET THIS SAME BUFFER BAD THINGS WILL HAPPEN gap = chunk.length / DOWN_FACTOR; currentChunk.clear(); Log.v("", "Inside While Loop Break Point 4a"); // ZERO CROSSINGS int increment = 1; if (chunk.length > 1000) { increment = (int) ((double) chunk.length / ((double) 1000)); } // Downsampling for (int i = 0; i < chunk.length; i = i + increment) { data.add((double) chunk[i]); currentChunk.add((double) chunk[i]); tempCount++; if (currentChunk.size() > 1) { iter += FastMath.abs(sign(currentChunk.get(currentChunk.size() - 1)) - sign(currentChunk.get(currentChunk.size() - 2))); } } increment = 0; tempCount = 0; zeroCrossings[count] = 0.5 * iter; po2 = FastMath.ceil(FastMath.log(currentChunk.size()) / FastMath.log(2)); input = new Complex[(int) (FastMath.pow(2.0, po2))]; Log.v("", "chunk length: " + chunk.length); Log.v("", "input length: " + input.length); for (int i = 0; i < input.length; i++) { if (i < currentChunk.size()) { input[i] = new Complex((double) currentChunk.get(i)); } else { input[i] = new Complex(0.0); } } // FFT output = transformer.transform(input, TransformType.FORWARD); outputExample.add(centroidDen); // CENTROID AND FLUX for (int i = 0; i < output.length; i++) { if (count > 0) { fftPrevious = fftValue; } fftValue = FastMath.hypot(output[i].getReal(), output[i].getImaginary()); fluxSquared += (fftValue - fftPrevious) * (fftValue - fftPrevious); centroidNum += i * fftValue; centroidDen += fftValue; } // for(int i = 0; i < output.length; i++){ // // normConst += FastMath.hypot(output[i].getReal(), output[i].getImaginary()) * // FastMath.hypot(output[i].getReal(), output[i].getImaginary()); // // // } // fluxSquared = fluxSquared / normConst; flux[count] = FastMath.sqrt(fluxSquared) / 1000.0; // ROLLOFF while (!foundRolloff && r < output.length - 1) { r++; sum += FastMath.hypot(output[r].getReal(), output[r].getImaginary()); foundRolloff = checkRolloff(ROLLOFF_PROPORTIONAL_ERROR, sum, centroidDen); } fftSums[count] = centroidDen; if (centroidDen != 0.0) { centroid[count] = centroidNum / centroidDen; } else { centroid[count] = 0.0; } rolloff[count] = r; iter = 0; fluxSquared = 0.0; centroidNum = 0.0; centroidDen = 0.0; r = 0; sum = 0.0; foundRolloff = false; count++; //audioTrack.write(chunk, info.offset, info.offset + info.size); // AudioTrack write data decoder.releaseOutputBuffer(outIndex, false); break; } // All decoded frames have been rendered, we can stop playing now if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) { Log.d("DecodeActivity", "OutputBuffer BUFFER_FLAG_END_OF_STREAM"); break; } if (count > 0) { previousOutput = output; output = null; } } if (count == NUM_CHUNKS) { // Log.v("", "Timestamp of last chunk: " + String.valueOf(extractor.getSampleTime())); decoder.stop(); decoder.release(); extractor.release(); } } // while loop currentChunk.clear(); currentChunk = null; // for(int i = 0; i < centroid.length; i++){ // Log.v("", "centroid: " + String.valueOf(centroid[i])); // } double energySum = 0.0; double energyAverage = 0.0; int lowEnergyCount = 0; for (int i = 0; i < NUM_CHUNKS; i++) { energySum += fftSums[i]; } energyAverage = energySum / NUM_CHUNKS; for (int i = 0; i < NUM_CHUNKS; i++) { if (fftSums[i] < energyAverage) { lowEnergyCount++; } } lowEnergy = 100.0 * (((double) lowEnergyCount) / ((double) NUM_CHUNKS)); // Work out the means and standard deviations for (int i = 0; i < NUM_CHUNKS; i++) { fluxMean += flux[i]; zeroCrossingsMean += zeroCrossings[i]; centroidMean += centroid[i]; rolloffMean += rolloff[i]; } fluxMean = fluxMean / flux.length; zeroCrossingsMean = zeroCrossingsMean / zeroCrossings.length; centroidMean = centroidMean / centroid.length; rolloffMean = rolloffMean / rolloff.length; for (int i = 0; i < NUM_CHUNKS; i++) { fluxStdDeviation += (flux[i] - fluxMean) * (flux[i] - fluxMean); zeroCrossingsStdDeviation += (zeroCrossings[i] - zeroCrossingsMean) * (zeroCrossings[i] - zeroCrossingsMean); centroidStdDeviation += (centroid[i] - centroidMean) * (centroid[i] - centroidMean); rolloffStdDeviation += (rolloff[i] - rolloffMean) * (rolloff[i] - rolloffMean); } fluxStdDeviation = Math.sqrt(fluxStdDeviation / flux.length); zeroCrossingsStdDeviation = Math.sqrt(zeroCrossingsStdDeviation / zeroCrossings.length); centroidStdDeviation = Math.sqrt(centroidStdDeviation / centroid.length); rolloffStdDeviation = Math.sqrt(rolloffStdDeviation / rolloff.length); Log.v("", "fluxMean: " + String.valueOf(fluxMean)); Log.v("", "zeroCrossingsMean: " + String.valueOf(zeroCrossingsMean)); Log.v("", "centroidMean: " + String.valueOf(centroidMean)); Log.v("", "rolloffMean: " + String.valueOf(rolloffMean)); Log.v("", "fluxStdDeviation: " + String.valueOf(fluxStdDeviation)); Log.v("", "zeroCrossingsStdDeviation: " + String.valueOf(zeroCrossingsStdDeviation)); Log.v("", "centroidStdDeviation: " + String.valueOf(centroidStdDeviation)); Log.v("", "rolloffStdDeviation: " + String.valueOf(rolloffStdDeviation)); Log.v("", "lowEnergy: " + String.valueOf(lowEnergy)); Log.v("", "data size: " + String.valueOf(data.size())); // BEAT ANALYSIS Transform t = new Transform(new FastWaveletTransform(new Daubechies4())); double[] dataArray = new double[data.size()]; for (int i = 0; i < data.size(); i++) { dataArray[i] = data.get(i); } data.clear(); data = null; double powerOf2 = FastMath.ceil(FastMath.log(chunk.length) / FastMath.log(2)); double[] dataArrayPo2 = Arrays.copyOf(dataArray, (int) (FastMath.pow(2.0, powerOf2))); dataArray = null; double[] dataCurrentInputArray = null; double[] dataCurrentOutputArray = null; double[] dataCumulativeArray = new double[dataArrayPo2.length]; for (int i = 0; i < dataCumulativeArray.length; i++) { dataCumulativeArray[i] = 0.0; } double temp1 = 0.0; double temp2 = 0.0; ArrayList<Double> tempList = new ArrayList<Double>(); int k = 16; // Downsampling factor int tempCount1 = 0; double mean = 0.0; for (int level = 0; level < (int) FastMath.log(2.0, dataArrayPo2.length); level++) { dataCurrentInputArray = t.forward(dataArrayPo2, level); dataCurrentOutputArray = dataCurrentInputArray; dataCurrentOutputArray[0] = 0.0; for (int i = 1; i < dataCurrentOutputArray.length; i++) { temp1 = FastMath.abs(dataCurrentInputArray[i]); // Full-wave rectification dataCurrentOutputArray[i] = (1.0 - ALPHA) * temp1 - ALPHA * dataCurrentOutputArray[i - 1]; // Low-pass filtering } tempCount1 = 0; mean = 0.0; while (k * tempCount1 < dataCurrentOutputArray.length) { tempList.add(dataCurrentOutputArray[k * tempCount1]); // Downsampling by k mean += dataCurrentOutputArray[k * tempCount1]; tempCount1++; } mean = mean / dataCurrentOutputArray.length; tempCount1 = 0; while (k * tempCount1 < dataCurrentOutputArray.length) { dataCumulativeArray[k * tempCount1] += tempList.get(tempCount1) - mean; // Mean removal tempCount1++; } } int N = dataCumulativeArray.length; ArrayList<Double> dataList = new ArrayList<Double>(); double dataElement = 0.0; for (int i = 0; i < N; i++) { if (dataCumulativeArray[i] != 0.0) { dataElement = autocorrelate(i, N, dataCumulativeArray); dataList.add(dataElement); Log.v("", "dataList: " + String.valueOf(dataElement)); } } PeakDetector peakDetector = new PeakDetector(dataList); int[] peakIndices = peakDetector.process(5, 2); HashSet<Integer> hs = new HashSet<Integer>(); for (int i = 0; i < peakIndices.length; i++) { hs.add(peakIndices[i]); } ArrayList<Integer> indicesList = new ArrayList<Integer>(); ArrayList<Double> valuesList = new ArrayList<Double>(); indicesList.addAll(hs); Double tempDoub = 0.0; HashMap<Double, Integer> hm = new HashMap<Double, Integer>(); for (int i = 0; i < indicesList.size(); i++) { tempDoub = dataList.get(indicesList.get(i)); hm.put(tempDoub, indicesList.get(i)); } indicesList.clear(); valuesList.clear(); Entry<Double, Integer> tempEntry = null; Iterator<Entry<Double, Integer>> it = hm.entrySet().iterator(); while (it.hasNext()) { tempEntry = (Entry<Double, Integer>) it.next(); if (tempEntry.getValue() < 75) { it.remove(); } else { //indicesList.add(tempEntry.getValue()); valuesList.add(tempEntry.getKey()); } } Collections.sort(valuesList); for (int i = 0; i < valuesList.size(); i++) { indicesList.add(hm.get(valuesList.get(i))); } double valuesSum = 0.0; double histogramSum = 0.0; double beatStrength = 0.0; double P1 = 0.0; double P2 = 0.0; double A1 = 0.0; double A2 = 0.0; double RA = 0.0; for (int i = 0; i < dataList.size(); i++) { histogramSum += dataList.get(i); } for (int i = 0; i < valuesList.size(); i++) { valuesSum += valuesList.get(i); } // if(histogramSum != 0.0 && valuesList.size() != 0){ // SUM = (1000.0 * valuesSum) / (histogramSum * valuesList.size()); // } if (valuesList.size() != 0) { beatStrength = valuesSum / valuesList.size(); } if (indicesList.size() > 0) { // Set P1 as the largest peak P1 = (double) indicesList.get(indicesList.size() - 1); } if (indicesList.size() > 1) { int beatCount = indicesList.size() - 2; boolean beatFound = false; // Start with P2 as the second largest peak P2 = (double) indicesList.get(indicesList.size() - 2); double diff = 0; // Iterate backwards through the peaks, largest to smallest while (!beatFound && beatCount > -1) { diff = ((double) indicesList.get(beatCount)) - P1; if (FastMath.abs(diff) / P1 > 0.3) { // Set P2 as the period of the first peak that is reasonably different from P1 P2 = (double) indicesList.get(beatCount); beatFound = true; } beatCount--; } } if (indicesList.size() > 0) { A1 = FastMath.abs(dataList.get((int) P1)) / histogramSum; if (P2 != 0.0) { A2 = FastMath.abs(dataList.get((int) P2)) / histogramSum; } if (A1 != 0.0) { RA = A2 / A1; } } for (int i = 0; i < valuesList.size(); i++) { Log.v("", String.valueOf(i) + ") valuesList: " + String.valueOf(valuesList.get(i))); } Log.v("", "P1: " + String.valueOf(P1)); Log.v("", "P2: " + String.valueOf(P2)); Log.v("", "A1: " + String.valueOf(A1)); Log.v("", "A2: " + String.valueOf(A2)); Log.v("", "RA: " + String.valueOf(RA)); Log.v("", "SUM: " + String.valueOf(histogramSum)); Log.v("", "Number of Peaks: " + String.valueOf(valuesList.size())); double[] result = { fluxMean, zeroCrossingsMean, centroidMean, rolloffMean, fluxStdDeviation, zeroCrossingsStdDeviation, centroidStdDeviation, rolloffStdDeviation, lowEnergy, P1, P2, A1, A2, RA, histogramSum, valuesList.size() }; final DenseInstance denseInstance = new DenseInstance(result); if (P1 + P2 + A1 + A2 + RA != 0.0) { Handler handler = new Handler(Looper.getMainLooper()); handler.post(new ReturnResultsRunnable(lock, mAudioCallback, denseInstance, classLabel)); } else { Log.v("", "Track could not be classified!"); } // for(int i = 0; i < dataList.size(); i++){ // Log.v("", String.valueOf(i) + ") autocorrelation: " + String.valueOf(dataList.get(i))); // histogramSum += dataList.get(i); // } // Log.v("", "indicesList size: " + String.valueOf(indicesList.size())); // for(int i = 0; i < valuesList.size(); i++){ // Log.v("", "indicesList: " + String.valueOf(indicesList.get(i)) + ", value: " + String.valueOf(valuesList.get(i))); // valuesSum += valuesList.get(i); // } //Classifier c = new KNearestNeighbors(5); // double A0 = valuesList.get(valuesList.size() - 1) / valuesSum; // double A1 = valuesList.get(valuesList.size() - 2) / valuesSum; // double RA = A1 / A0; // double P0 = 1 / ((double) indicesList.get(indicesList.size() - 1)); // double P1 = 1 / ((double) indicesList.get(indicesList.size() - 2)); // // Log.v("", "A0: " + String.valueOf(A0)); // Log.v("", "A1: " + String.valueOf(A1)); // Log.v("", "RA: " + String.valueOf(RA)); // Log.v("", "P0: " + String.valueOf(P0)); // Log.v("", "P1: " + String.valueOf(P1)); // Log.v("", "SUM: " + String.valueOf(histogramSum)); long durationUs = SystemClock.elapsedRealtimeNanos() - start; double durationSecs = ((double) durationUs) / 1000000000.0; Log.v("", "count = " + String.valueOf(count) + ", Sample rate: " + String.valueOf(sampleRate) + ", Duration: " + String.valueOf(durationSecs)); // audioTrack.stop(); // audioTrack.release(); // audioTrack = null; }