com.microsoft.mimickeralarm.mimics.MimicTongueTwisterFragment.java Source code

Java tutorial

Introduction

Here is the source code for com.microsoft.mimickeralarm.mimics.MimicTongueTwisterFragment.java

Source

/*
 *
 * Copyright (c) Microsoft. All rights reserved.
 * Licensed under the MIT license.
 *
 * Project Oxford: http://ProjectOxford.ai
 *
 * Project Oxford Mimicker Alarm Github:
 * https://github.com/Microsoft/ProjectOxford-Apps-MimickerAlarm
 *
 * Copyright (c) Microsoft Corporation
 * All rights reserved.
 *
 * MIT License:
 * Permission is hereby granted, free of charge, to any person obtaining
 * a copy of this software and associated documentation files (the
 * "Software"), to deal in the Software without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Software, and to
 * permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be
 * included in all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED ""AS IS"", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 *
 */

package com.microsoft.mimickeralarm.mimics;

import android.content.Context;
import android.content.res.Resources;
import android.graphics.Bitmap;
import android.graphics.Canvas;
import android.net.Uri;
import android.os.Bundle;
import android.support.annotation.Nullable;
import android.support.v4.app.Fragment;
import android.support.v4.content.ContextCompat;
import android.util.Log;
import android.view.LayoutInflater;
import android.view.View;
import android.view.ViewGroup;
import android.widget.LinearLayout;
import android.widget.TextView;

import com.microsoft.mimickeralarm.R;
import com.microsoft.mimickeralarm.mimics.MimicFactory.MimicResultListener;
import com.microsoft.mimickeralarm.ringing.ShareFragment;
import com.microsoft.mimickeralarm.utilities.Loggable;
import com.microsoft.mimickeralarm.utilities.Logger;
import com.microsoft.mimickeralarm.utilities.KeyUtilities;
import com.microsoft.projectoxford.speechrecognition.Confidence;
import com.microsoft.projectoxford.speechrecognition.ISpeechRecognitionServerEvents;
import com.microsoft.projectoxford.speechrecognition.MicrophoneRecognitionClient;
import com.microsoft.projectoxford.speechrecognition.RecognitionResult;
import com.microsoft.projectoxford.speechrecognition.RecognitionStatus;
import com.microsoft.projectoxford.speechrecognition.RecognizedPhrase;
import com.microsoft.projectoxford.speechrecognition.SpeechRecognitionMode;
import com.microsoft.projectoxford.speechrecognition.SpeechRecognitionServiceFactory;

import java.util.Random;

/**
 * Implements the UI and logic of the Tongue Twister mimic game
 *
 * on start randomly selects one of the tongue twisters
 * when the user presses the record button, uses the speech SDK binaries to capture and send audio
 * to the Project Oxford speech API.
 *
 * There are two types of results returned. Partial and Final.
 * Final result is returned by Project Oxford API. It contains the most likely Speech->Text transcription
 * of the provided audio. It is used here to compute the final correctness
 *
 * Partial results are continuously returned by native code. It is fast but the quality is not as good
 * as the final result. It is used here to continuously provide feedback to the user
 *
 *
 * The correctness is computed by a simple Levenshtein distance calculation between the question
 * tongue twister and the final result returned by Project Oxford.
 */
public class MimicTongueTwisterFragment extends Fragment
        implements ISpeechRecognitionServerEvents, IMimicImplementation {
    private final static int TIMEOUT_MILLISECONDS = 20000;
    private final static float DIFFERENCE_SUCCESS_THRESHOLD = 0.3f;
    private final static float DIFFERENCE_PERFECT_THRESHOLD = 0.1f;
    private static String LOGTAG = "MimicTongueTwisterFragment";
    MimicResultListener mCallback;
    private MicrophoneRecognitionClient mMicClient = null;
    private SpeechRecognitionMode mRecognitionMode;
    private String mUnderstoodText = null;
    private String mQuestion = null;
    private TextView mTextResponse;
    private String mSuccessMessage;
    private Uri mSharableUri;
    private IMimicMediator mStateManager;

    @Nullable
    @Override
    public View onCreateView(LayoutInflater inflater, ViewGroup container, Bundle savedInstanceState) {
        View view = inflater.inflate(R.layout.fragment_tongue_twister_mimic, container, false);
        ProgressButton progressButton = (ProgressButton) view.findViewById(R.id.capture_button);
        progressButton.setReadyState(ProgressButton.State.ReadyAudio);

        mStateManager = new MimicStateManager();
        mStateManager.registerCountDownTimer((CountDownTimerView) view.findViewById(R.id.countdown_timer),
                TIMEOUT_MILLISECONDS);
        mStateManager.registerStateBanner((MimicStateBanner) view.findViewById(R.id.mimic_state));
        mStateManager.registerProgressButton(progressButton, MimicButtonBehavior.AUDIO);
        mStateManager.registerMimic(this);

        initialize(view);

        Logger.init(getContext());
        Loggable.UserAction userAction = new Loggable.UserAction(Loggable.Key.ACTION_GAME_TWISTER);
        Logger.track(userAction);

        return view;
    }

    @Override
    public void onAttach(Context context) {
        super.onAttach(context);
        mCallback = (MimicResultListener) context;
    }

    @Override
    public void onDetach() {
        super.onDetach();
        mCallback = null;
    }

    @Override
    public void onStart() {
        super.onStart();
        mStateManager.start();
    }

    @Override
    public void onStop() {
        super.onStop();
        mStateManager.stop();
    }

    @Override
    public void onDestroy() {
        super.onDestroy();
        Logger.flush();
    }

    @Override
    public void onPartialResponseReceived(String s) {
        Log.d(LOGTAG, s);
        mTextResponse.setText(s);
        mUnderstoodText = s;
    }

    @Override
    public void onFinalResponseReceived(RecognitionResult response) {
        if (mStateManager.isMimicRunning()) {
            boolean isFinalDictationMessage = mRecognitionMode == SpeechRecognitionMode.LongDictation
                    && (response.RecognitionStatus == RecognitionStatus.EndOfDictation
                            || response.RecognitionStatus == RecognitionStatus.DictationEndSilenceTimeout);
            if (mRecognitionMode == SpeechRecognitionMode.ShortPhrase || isFinalDictationMessage) {
                mMicClient.endMicAndRecognition();
                for (RecognizedPhrase res : response.Results) {
                    Log.d(LOGTAG, String.valueOf(res.Confidence));
                    Log.d(LOGTAG, String.valueOf(res.DisplayText));

                    if (res.Confidence == Confidence.Normal) {
                        mUnderstoodText = res.DisplayText;
                    } else if (res.Confidence == Confidence.High) {
                        mUnderstoodText = res.DisplayText;
                        break;
                    }
                }

                mTextResponse.setText(mUnderstoodText);
                verify();
            }
        }
    }

    @Override
    public void onIntentReceived(final String s) {
        Log.d(LOGTAG, s);
    }

    @Override
    public void onError(int errorCode, final String s) {
        Loggable.AppError error = new Loggable.AppError(Loggable.Key.APP_ERROR, s);
        Logger.track(error);
    }

    @Override
    public void onAudioEvent(boolean recording) {
        if (!recording) {
            stopCapture();
        }
    }

    @Override
    public void initializeCapture() {
        mRecognitionMode = SpeechRecognitionMode.ShortPhrase;
        try {
            //TODO: localize
            String language = "en-us";
            String subscriptionKey = KeyUtilities.getToken(getActivity(), "speech");
            if (mMicClient == null) {
                mMicClient = SpeechRecognitionServiceFactory.createMicrophoneClient(getActivity(), mRecognitionMode,
                        language, this, subscriptionKey);
            }
        } catch (Exception e) {
            Logger.trackException(e);
        }
    }

    @Override
    public void startCapture() {
        mMicClient.startMicAndRecognition();
    }

    @Override
    public void stopCapture() {
        if (mMicClient != null) {
            mMicClient.endMicAndRecognition();
        }
    }

    @Override
    public void onCountDownTimerExpired() {
        gameFailure(false);
    }

    @Override
    public void onSucceeded() {
        if (mCallback != null) {
            mCallback.onMimicSuccess(mSharableUri.getPath());
        }
    }

    @Override
    public void onFailed() {
        if (mCallback != null) {
            mCallback.onMimicFailure();
        }
    }

    protected void gameSuccess(double difference) {
        mSuccessMessage = getString(R.string.mimic_success_message);
        if (difference <= DIFFERENCE_PERFECT_THRESHOLD) {
            mSuccessMessage = getString(R.string.mimic_twister_perfect_message);
        }
        createSharableBitmap();
        mStateManager.onMimicSuccess(mSuccessMessage);
    }

    protected void gameFailure(boolean allowRetry) {
        if (allowRetry) {
            String failureMessage = getString(R.string.mimic_failure_message);
            mStateManager.onMimicFailureWithRetry(failureMessage);
        } else {
            Loggable.UserAction userAction = new Loggable.UserAction(Loggable.Key.ACTION_GAME_TWISTER_TIMEOUT);
            userAction.putProp(Loggable.Key.PROP_QUESTION, mQuestion);
            Logger.track(userAction);
            String failureMessage = getString(R.string.mimic_time_up_message);
            mStateManager.onMimicFailure(failureMessage);
        }
    }

    private void initialize(View view) {
        mTextResponse = (TextView) view.findViewById(R.id.understood_text);
        generateQuestion(view);
    }

    private void generateQuestion(View view) {
        Resources resources = getResources();
        String[] questions = resources.getStringArray(R.array.tongue_twisters);
        mQuestion = questions[new Random().nextInt(questions.length)];

        final TextView instructionTextView = (TextView) view.findViewById(R.id.instruction_text);
        instructionTextView.setText(mQuestion);
    }

    //
    // Create bitmap for sharing
    //
    private void createSharableBitmap() {
        Bitmap sharableBitmap = Bitmap.createBitmap(getView().getWidth(), getView().getHeight(),
                Bitmap.Config.ARGB_8888);
        Canvas canvas = new Canvas(sharableBitmap);
        canvas.drawColor(ContextCompat.getColor(getContext(), R.color.white));

        // Load the view for the sharable. This will be drawn to the bitmap
        LayoutInflater inflater = (LayoutInflater) getContext().getSystemService(Context.LAYOUT_INFLATER_SERVICE);
        LinearLayout layout = (LinearLayout) inflater.inflate(R.layout.fragment_sharable_tongue_twister, null);

        TextView textView = (TextView) layout.findViewById(R.id.twister_sharable_tongue_twister);
        textView.setText(mQuestion);

        textView = (TextView) layout.findViewById(R.id.twister_sharable_i_said);
        textView.setText(mUnderstoodText);

        textView = (TextView) layout.findViewById(R.id.mimic_twister_share_success);
        textView.setText(mSuccessMessage);

        // Perform the layout using the dimension of the bitmap
        int widthSpec = View.MeasureSpec.makeMeasureSpec(canvas.getWidth(), View.MeasureSpec.EXACTLY);
        int heightSpec = View.MeasureSpec.makeMeasureSpec(canvas.getHeight(), View.MeasureSpec.EXACTLY);
        layout.measure(widthSpec, heightSpec);
        layout.layout(0, 0, layout.getMeasuredWidth(), layout.getMeasuredHeight());

        // Draw the generated view to canvas
        layout.draw(canvas);

        String title = getString(R.string.app_short_name) + ": " + getString(R.string.mimic_twister_name);
        mSharableUri = ShareFragment.saveShareableBitmap(getActivity(), sharableBitmap, title);
    }

    //https://en.wikibooks.org/wiki/Algorithm_Implementation/Strings/Levenshtein_distance
    private int levenshteinDistance(CharSequence lhs, CharSequence rhs) {
        if (lhs == null && rhs == null) {
            return 0;
        }
        if (lhs == null) {
            return rhs.length();
        }
        if (rhs == null) {
            return lhs.length();
        }

        int len0 = lhs.length() + 1;
        int len1 = rhs.length() + 1;

        // the array of distances
        int[] cost = new int[len0];
        int[] newcost = new int[len0];

        // initial cost of skipping prefix in String s0
        for (int i = 0; i < len0; i++)
            cost[i] = i;

        // dynamically computing the array of distances

        // transformation cost for each letter in s1
        for (int j = 1; j < len1; j++) {
            // initial cost of skipping prefix in String s1
            newcost[0] = j;

            // transformation cost for each letter in s0
            for (int i = 1; i < len0; i++) {
                // matching current letters in both strings
                int match = (lhs.charAt(i - 1) == rhs.charAt(j - 1)) ? 0 : 1;

                // computing cost for each transformation
                int cost_replace = cost[i - 1] + match;
                int cost_insert = cost[i] + 1;
                int cost_delete = newcost[i - 1] + 1;

                // keep minimum cost
                newcost[i] = Math.min(Math.min(cost_insert, cost_delete), cost_replace);
            }

            // swap cost/newcost arrays
            int[] swap = cost;
            cost = newcost;
            newcost = swap;
        }

        // the distance is the cost for transforming all letters in both strings
        return cost[len0 - 1];
    }

    private void verify() {
        if (mUnderstoodText == null) {
            gameFailure(true);
            return;
        }

        double difference = (double) levenshteinDistance(mUnderstoodText, mQuestion) / (double) mQuestion.length();

        Loggable.UserAction userAction = new Loggable.UserAction(Loggable.Key.ACTION_GAME_TWISTER_SUCCESS);
        userAction.putProp(Loggable.Key.PROP_QUESTION, mQuestion);
        userAction.putProp(Loggable.Key.PROP_DIFF, difference);

        if (difference <= DIFFERENCE_SUCCESS_THRESHOLD) {
            Logger.track(userAction);
            gameSuccess(difference);
        } else {
            userAction.Name = Loggable.Key.ACTION_GAME_TWISTER_FAIL;
            Logger.track(userAction);
            gameFailure(true);
        }
    }
}