com.hichinaschool.flashcards.libanki.Media.java Source code

Java tutorial

Introduction

Here is the source code for com.hichinaschool.flashcards.libanki.Media.java

Source

/****************************************************************************************
 * Copyright (c) 2011 Kostas Spyropoulos <inigo.aldana@gmail.com>                       *
 *                                                                                      *
 * This program is free software; you can redistribute it and/or modify it under        *
 * the terms of the GNU General Public License as published by the Free Software        *
 * Foundation; either version 3 of the License, or (at your option) any later           *
 * version.                                                                             *
 *                                                                                      *
 * This program is distributed in the hope that it will be useful, but WITHOUT ANY      *
 * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A      *
 * PARTICULAR PURPOSE. See the GNU General Public License for more details.             *
 *                                                                                      *
 * You should have received a copy of the GNU General Public License along with         *
 * this program.  If not, see <http://www.gnu.org/licenses/>.                           *
 ****************************************************************************************/

package com.hichinaschool.flashcards.libanki;

import android.database.Cursor;
import android.database.SQLException;
import android.net.Uri;
import android.util.Log;

import com.hichinaschool.flashcards.anki.AnkiDatabaseManager;
import com.hichinaschool.flashcards.anki.AnkiDb;
import com.hichinaschool.flashcards.anki.AnkiDroidApp;
import com.hichinaschool.flashcards.anki.Pair;

import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;

import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.zip.ZipEntry;
import java.util.zip.ZipException;
import java.util.zip.ZipFile;
import java.util.zip.ZipOutputStream;

/**
 * Class with static functions related with media handling (images and sounds).
 */
public class Media {
    public static final int MEDIA_ADD = 0;
    public static final int MEDIA_REM = 1;
    public static final long SYNC_ZIP_SIZE = 2560 * 1024;

    public static final Pattern fMediaRegexps[] = { Pattern.compile("(?i)(\\[sound:([^]]+)\\])"),
            Pattern.compile("(?i)(<img[^>]+src=[\"']?([^\"'>]+)[\"']?[^>]*>)") };
    private static final Pattern fSoundRegexps = Pattern.compile("\\[sound:(.*?)\\]");
    private static final Pattern fRemoteFilePattern = Pattern.compile("(https?|ftp)://");
    private static final Pattern fDangerousCharacters = Pattern.compile("[]\\[<>:/\\\\&?\\\"\\|]");
    private static final Pattern fFileOrdinal = Pattern.compile(" \\((\\d+)\\)$");

    private Collection mCol;
    private String mDir;
    private String mMediaDbFilename;
    private AnkiDb mMediaDb;

    public Media(Collection col, boolean server) {
        mCol = col;
        if (server) {
            mDir = null;
            return;
        }
        mDir = col.getPath().replaceFirst("\\.anki2$", ".media");
        mMediaDbFilename = mDir + ".db";
        File fd = new File(mDir);
        if (!fd.exists()) {
            if (!fd.mkdir()) {
                Log.e(AnkiDroidApp.TAG, "Cannot create media directory: " + mDir);
            }
        }

        connect();
    }

    public void connect() {
        File mediaDbFile = new File(mMediaDbFilename);
        if (!mediaDbFile.exists()) {
            // Copy an empty collection file from the assets to the SD card.
            InputStream stream;
            try {
                stream = AnkiDroidApp.getAppResources().getAssets().open("collection.media.db");
                Utils.writeToFile(stream, mMediaDbFilename);
                stream.close();
            } catch (IOException e) {
                throw new RuntimeException(e);
            }
        }
        mMediaDb = AnkiDatabaseManager.getDatabase(mMediaDbFilename);
    }

    public void close() {
        AnkiDatabaseManager.closeDatabase(mMediaDbFilename);
        mMediaDb = null;
        mCol = null;
    }

    public String getDir() {
        return mDir;
    }

    // Adding media
    // /////////////

    /**
     * Copy PATH to MEDIADIR and return new filename. If the same name exists, compare checksums.
     * 
     * @param opath The path where the media file exists before adding it.
     * @return The filename of the resulting file.
     */
    public String addFile(String opath) {
        String mdir = getDir();
        // remove any dangerous characters
        String base = fDangerousCharacters.matcher(new File(opath).getName()).replaceAll("");
        // if it doesn't exist, copy it directly
        File newMediaFile = new File(mdir, base);
        String dst = newMediaFile.getAbsolutePath();
        if (!newMediaFile.exists()) {
            try {
                Utils.copyFile(new File(opath), newMediaFile);
            } catch (IOException e) {
                throw new RuntimeException(e);
            }
            return base;
        }
        if (filesIdentical(opath, dst)) {
            return base;
        }
        // otherwise, find a unique name
        String root, ext;
        int extIndex = base.lastIndexOf('.');
        if (extIndex == 0 || extIndex == -1) {
            root = base;
            ext = "";
        } else {
            root = base.substring(0, extIndex);
            ext = base.substring(extIndex);
        }
        StringBuilder sb = null;
        String path = null;
        Matcher m = null;
        int n = 0;
        while (true) {
            sb = new StringBuilder(mdir);
            path = sb.append(File.separatorChar).append(root).append(ext).toString();
            newMediaFile = new File(path);
            if (!newMediaFile.exists()) {
                break;
            }
            m = fFileOrdinal.matcher(root);
            if (!m.find()) {
                root = root.concat(" (1)");
            } else {
                n = Integer.parseInt(m.group(1));
                root = m.replaceFirst(" (" + String.valueOf(n + 1) + ")");
            }
        }
        // copy and return
        try {
            Utils.copyFile(new File(opath), newMediaFile);
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
        return newMediaFile.getName();
    }

    /**
     * Checks if two files are identical
     * 
     * @param filepath1 The path of the first file to be checked
     * @param filepath2 The path of the second file to be checked
     * @return True if both files have the same contents
     */
    private boolean filesIdentical(String filepath1, String filepath2) {
        return Utils.fileChecksum(filepath1).equals(Utils.fileChecksum(filepath2));
    }

    // String manipulation
    // ////////////////////

    public List<String> filesInStr(String string) {
        return filesInStr(string, false);
    }

    /**
     * Extract media filenames from an HTML string.
     * 
     * @param string The string to scan for media filenames ([sound:...] or <img...>).
     * @param includeRemote If true will also include external http/https/ftp urls.
     * @return A list containing all the sound and image filenames found in the input string.
     */
    public List<String> filesInStr(String string, boolean includeRemote) {
        List<String> l = new ArrayList<String>();
        // Convert latex first
        string = LaTeX.mungeQA(string, mCol);
        // Extract filenames
        Matcher m = null;
        for (Pattern p : fMediaRegexps) {
            m = p.matcher(string);
            while (m.find()) {
                String fname = m.group(2);
                if (includeRemote || (!fRemoteFilePattern.matcher(fname.toLowerCase()).find())) {
                    l.add(fname);
                }
            }
        }
        return l;
    }

    /**
     * Strips a string from media references.
     * 
     * @param txt The string to be cleared of media references.
     * @return The media-free string.
     */
    public String strip(String txt) {
        Matcher m = null;
        for (Pattern p : fMediaRegexps) {
            m = p.matcher(txt);
            txt = m.replaceAll("");
        }
        return txt;
    }

    public String stripAudio(String txt) {
        Matcher m = fSoundRegexps.matcher(txt);
        return m.replaceAll("");
    }

    /**
     * Percent-escape UTF-8 characters in local image filenames.
     * 
     * @param string The string to search for image references and escape the filenames.
     * @return The string with the filenames of any local images percent-escaped as UTF-8.
     */
    public String escapeImages(String string) {
        Matcher m = fMediaRegexps[1].matcher(string);
        StringBuffer sb = new StringBuffer();
        while (m.find()) {
            if (fRemoteFilePattern.matcher(m.group(2)).find()) {
                m.appendReplacement(sb, m.group());
            } else {
                String tagBegin = m.group(1).substring(0, m.start(2));
                String fname = m.group(2);
                String tagEnd = m.group(1).substring(m.end(2));
                String tag = tagBegin + Uri.encode(fname) + tagEnd;
                m.appendReplacement(sb, tag);
            }
        }
        m.appendTail(sb);
        return sb.toString();
    }

    // Rebuilding DB
    // //////////////

    /**
     * Finds missing and unused media files
     * 
     * @return A list containing two lists of filenames (missingList, unusedList)
     */
    public List<List<String>> check() {
        File mdir = new File(getDir());
        List<List<String>> result = new ArrayList<List<String>>();
        List<String> unused = new ArrayList<String>();

        Set<String> normrefs = new HashSet<String>();
        for (String f : allMedia()) {
            if (AnkiDroidApp.SDK_VERSION > 9) {
                f = AnkiDroidApp.getCompat().normalizeUnicode(f);
            }
            normrefs.add(f);
        }
        for (File file : mdir.listFiles()) {
            if (file.isDirectory()) {
                continue;
            }
            if (file.getName().startsWith("_")) {
                // leading _ says to ignore file
                continue;
            }
            String nfile = file.getName();
            if (AnkiDroidApp.SDK_VERSION > 9) {
                nfile = AnkiDroidApp.getCompat().normalizeUnicode(nfile);
            }
            if (!normrefs.contains(nfile)) {
                unused.add(file.getName());
            } else {
                normrefs.remove(nfile);
            }
        }
        List<String> nohave = new ArrayList<String>(normrefs);
        result.add(nohave);
        result.add(unused);

        return result;
    }

    /**
     * Return a list of all referenced filenames.
     * 
     * @return The list of all media references found in the media database.
     */
    private List<String> allMedia() {
        Set<String> files = new HashSet<String>();
        List<String> fldsList = mCol.getDb().queryColumn(String.class, "select flds from notes", 0);
        for (String flds : fldsList) {
            List<String> fList = filesInStr(flds);
            for (String f : fList) {
                files.add(f);
            }
        }
        return new ArrayList<String>(files);
    }

    // Copying on import
    ////////////////////

    public boolean have(String fname) {
        return new File(fname, getDir()).exists();
    }

    // Media syncing - changes and removal
    // ////////////////////////////////////

    public boolean hasChanged() {
        return (mMediaDb != null && mMediaDb.queryLongScalar("select 1 from log limit 1", false) == 1);
    }

    public List<String> removed() {
        String sql = "select fname from log where type = " + Integer.toString(MEDIA_REM);
        return mMediaDb.queryColumn(String.class, sql, 0);
    }

    /**
     * Remove provided deletions and all locally-logged deletions, as server has acked them
     * 
     * @param fnames The list of filenames to be deleted.
     */
    public void syncRemove(JSONArray fnames) {
        mMediaDb.getDatabase().beginTransaction();
        try {
            for (int i = 0; i < fnames.length(); ++i) {
                String f = fnames.optString(i);
                if (f == "") {
                    continue;
                }
                File file = new File(getDir(), f);
                if (file.exists()) {
                    file.delete();
                }

                mMediaDb.execute("delete from log where fname = ?", new String[] { f });
                mMediaDb.execute("delete from media where fname = ?", new String[] { f });
            }
            mMediaDb.execute("delete from log where type = ?", new String[] { Integer.toString(MEDIA_REM) });
            mMediaDb.getDatabase().setTransactionSuccessful();
        } finally {

            mMediaDb.getDatabase().endTransaction();
        }
    }

    // Media syncing - unbundling zip files from server
    // /////////////////////////////////////////////////

    /**
     * Extract zip data.
     * 
     * @param zipData An input stream that represents a zipped file.
     * @return True if finished.
     */
    public boolean syncAdd(File zipData) {
        boolean finished = false;
        ZipFile z = null;
        ArrayList<Object[]> media = new ArrayList<Object[]>();
        long sizecnt = 0;
        JSONObject meta = null;
        int nextUsn = 0;
        try {
            z = new ZipFile(zipData, ZipFile.OPEN_READ);
            // get meta info first
            ZipEntry metaEntry = z.getEntry("_meta");
            // if (metaEntry.getSize() >= 100000) {
            // Log.e(AnkiDroidApp.TAG, "Size for _meta entry found too big (" + z.getEntry("_meta").getSize() + ")");
            // return false;
            // }
            meta = new JSONObject(Utils.convertStreamToString(z.getInputStream(metaEntry)));
            ZipEntry usnEntry = z.getEntry("_usn");
            String usnstr = Utils.convertStreamToString(z.getInputStream(usnEntry));
            nextUsn = Integer.parseInt(usnstr);
        } catch (JSONException e) {
            throw new RuntimeException(e);
        } catch (ZipException e) {
            throw new RuntimeException(e);
        } catch (IOException e) {
            throw new RuntimeException(e);
        }

        // Then loop through all files
        for (ZipEntry zentry : Collections.list(z.entries())) {
            // Check for zip bombs
            sizecnt += zentry.getSize();
            if (sizecnt > 100 * 1024 * 1024) {
                Log.e(AnkiDroidApp.TAG, "Media zip file exceeds 100MB uncompressed, aborting unzipping");
                return false;
            }
            if (zentry.getName().compareTo("_meta") == 0 || zentry.getName().compareTo("_usn") == 0) {
                // Ignore previously retrieved meta
                continue;
            } else if (zentry.getName().compareTo("_finished") == 0) {
                finished = true;
            } else {
                String name = meta.optString(zentry.getName());
                if (illegal(name)) {
                    continue;
                }
                String path = getDir().concat(File.separator).concat(name);
                try {
                    Utils.writeToFile(z.getInputStream(zentry), path);
                } catch (IOException e) {
                    throw new RuntimeException(e);
                }
                String csum = Utils.fileChecksum(path);
                // append db
                media.add(new Object[] { name, csum, _mtime(name) });
                mMediaDb.execute("delete from log where fname = ?", new String[] { name });
            }
        }

        // update media db and note new starting usn
        if (!media.isEmpty()) {
            mMediaDb.executeMany("insert or replace into media values (?,?,?)", media);
        }
        setUsn(nextUsn); // commits
        // if we have finished adding, we need to record the new folder mtime
        // so that we don't trigger a needless scan
        if (finished) {
            syncMod();
        }
        return finished;
    }

    /**
     * Check if the file name has illegal for the OS characters.
     * 
     * @param f The filename to be checked.
     * @return Returns true if at least an illegal character is found.
     */
    private boolean illegal(String f) {
        if (f.contains("/")) {
            return true;
        }
        return false;
    }

    // Media syncing - bundling zip files to send to server
    // Because there's no standard filename encoding for zips, and because not
    // all zip clients support retrieving mtime, we store the files as ascii
    // and place a json file in the zip with the necessary information.
    // /////////////////////////////////////////////////////

    /**
     * Add files to a zip until over SYNC_ZIP_SIZE. Return zip data.
     * 
     * @return Returns a tuple with two objects. The first one is the zip file contents, the second a list with the
     *         filenames of the files inside the zip.
     */
    public Pair<File, List<String>> zipAdded() {
        File f = new File(mCol.getPath().replaceFirst("collection\\.anki2$", "tmpSyncToServer.zip"));

        String sql = "select fname from log where type = " + Integer.toString(MEDIA_ADD);
        List<String> filenames = mMediaDb.queryColumn(String.class, sql, 0);
        List<String> fnames = new ArrayList<String>();

        try {
            ZipOutputStream zos = new ZipOutputStream(new BufferedOutputStream(new FileOutputStream(f)));
            zos.setLevel(8);

            JSONObject files = new JSONObject();
            int cnt = 0;
            long sz = 0;
            byte buffer[] = new byte[2048];
            boolean finished = true;
            for (String fname : filenames) {
                fnames.add(fname);
                File file = new File(getDir(), fname);
                BufferedInputStream bis = new BufferedInputStream(new FileInputStream(file), 2048);
                ZipEntry entry = new ZipEntry(Integer.toString(cnt));
                zos.putNextEntry(entry);
                int count = 0;
                while ((count = bis.read(buffer, 0, 2048)) != -1) {
                    zos.write(buffer, 0, count);
                }
                zos.closeEntry();
                bis.close();
                files.put(Integer.toString(cnt), fname);
                sz += file.length();
                if (sz > SYNC_ZIP_SIZE) {
                    finished = false;
                    break;
                }
                cnt += 1;
            }
            if (finished) {
                zos.putNextEntry(new ZipEntry("_finished"));
                zos.closeEntry();
            }
            zos.putNextEntry(new ZipEntry("_meta"));
            zos.write(Utils.jsonToString(files).getBytes());
            zos.close();
        } catch (FileNotFoundException e) {
            throw new RuntimeException(e);
        } catch (IOException e) {
            throw new RuntimeException(e);
        } catch (JSONException e) {
            throw new RuntimeException(e);
        }

        return new Pair<File, List<String>>(f, fnames);
    }

    /**
     * Remove records from log table in media DB for a list or files.
     * 
     * @param fnames A list containing the list of filenames to be removed from log table.
     */
    public void forgetAdded(List<String> fnames) {
        if (!fnames.isEmpty()) {
            ArrayList<Object[]> args = new ArrayList<Object[]>();
            for (String fname : fnames) {
                args.add(new Object[] { fname });
            }
            mMediaDb.executeMany("delete from log where fname = ?", args);
        }
    }

    // Tracking changes (private)
    // ///////////////////////////

    /**
     * Returns the number of seconds from epoch, since the last modification to the file in path.
     * 
     * @param path The path to the file we are checking.
     * @return The number of seconds (rounded down).
     */
    private long _mtime(String path) {
        File f = new File(path);
        return f.lastModified() / 1000;
    }

    private String _checksum(String path) {
        return Utils.fileChecksum(path);
    }

    public long usn() {
        return mMediaDb.queryLongScalar("select usn from meta");
    }

    public void setUsn(long usn) {
        mMediaDb.execute("update meta set usn = ?", new Object[] { usn });
    }

    private void syncMod() {
        mMediaDb.execute("update meta set dirMod = ?", new Object[] { _mtime(mDir) });
    }

    /**
     * Return dir mtime if it has changed since the last findChanges() Doesn't track edits, but user can add or remove a
     * file to update
     * 
     * @return The modification time of the media directory, if it has changed since the last call of findChanges(). If
     *         it hasn't, it returns 0.
     */
    public long _changed() {
        long mod = mMediaDb.queryLongScalar("select dirMod from meta");
        long mtime = _mtime(getDir());
        if (mod != 0 && mod == mtime) {
            return 0;
        }
        return mtime;
    }

    /**
     * Scan the media folder if it's changed, and note any changes.
     */
    public void findChanges() {
        if (_changed() != 0) {
            _logChanges();
        }
    }

    private void _logChanges() {
        Pair<List<String>, List<String>> result = _changes();
        ArrayList<Object[]> log = new ArrayList<Object[]>();
        ArrayList<Object[]> media = new ArrayList<Object[]>();
        ArrayList<Object[]> mediaRem = new ArrayList<Object[]>();

        for (String f : result.first) {
            long mt = _mtime(f);
            String csum = _checksum(getDir().concat(File.separator).concat(f));
            media.add(new Object[] { f, csum, mt });
            log.add(new Object[] { f, MEDIA_ADD });
        }
        for (String f : result.second) {
            mediaRem.add(new Object[] { f });
            log.add(new Object[] { f, MEDIA_REM });
        }

        // update media db
        mMediaDb.executeMany("insert or replace into media values (?, ?, ?)", media);
        if (mediaRem.size() > 0) {
            mMediaDb.executeMany("delete from media where fname = ?", mediaRem);
        }
        mMediaDb.execute("update meta set dirMod = ?", new Object[] { _mtime(getDir()) });
        // and logs
        mMediaDb.executeMany("insert or replace into log values (?, ?)", log);
    }

    private Pair<List<String>, List<String>> _changes() {
        Map<String, Pair<String, Long>> cache = new HashMap<String, Pair<String, Long>>();
        Map<String, Boolean> used = new HashMap<String, Boolean>();
        Cursor cur = null;
        try {
            cur = mMediaDb.getDatabase().query("media", new String[] { "fname", "csum", "mod" }, null, null, null,
                    null, null);
            while (cur.moveToNext()) {
                cache.put(cur.getString(0), new Pair<String, Long>(cur.getString(1), cur.getLong(1)));
                used.put(cur.getString(0), false);
            }
        } catch (SQLException e) {
            throw new RuntimeException(e);
        } finally {
            if (cur != null) {
                cur.close();
            }
        }

        List<String> added = new ArrayList<String>();
        List<String> removed = new ArrayList<String>();

        File mediaDir = new File(getDir());
        for (File f : mediaDir.listFiles()) {
            // ignore folders and thumbs.db
            if (f.isDirectory()) {
                continue;
            }
            String fname = f.getName();
            if (fname.compareTo("thumbs.db") == 0) {
                continue;
            }
            // and files with invalid chars
            boolean bad = false;
            for (String c : new String[] { "\0", "/", "\\", ":" }) {
                if (fname.contains(c)) {
                    bad = true;
                    break;
                }
            }
            if (bad) {
                continue;
            }
            // empty files are invalid; clean them up and continue
            if (f.length() == 0) {
                f.delete();
                continue;
            }
            // newly added?
            if (!cache.containsKey(fname)) {
                added.add(fname);
            } else {
                // modified since last time?
                if ((f.lastModified() / 1000) != cache.get(fname).second) {
                    // and has different checksum?
                    if (_checksum(f.getAbsolutePath()).compareTo(cache.get(fname).first) != 0) {
                        added.add(fname);
                    }
                }
                // mark as used
                used.put(fname, true);
            }
        }

        // look for any entries in the cache that no longer exist on disk
        for (String fname : used.keySet()) {
            if (!used.get(fname)) {
                removed.add(fname);
            }
        }
        return new Pair<List<String>, List<String>>(added, removed);
    }

    public Pair<Long, Long> sanityCheck() {
        return new Pair<Long, Long>(mMediaDb.queryLongScalar("select count() from log"),
                mMediaDb.queryLongScalar("select count() from media"));
    }

    public void resetMediaDb() {
        mMediaDb.execute("delete from log");
        mMediaDb.execute("delete from media");
        mMediaDb.execute("delete from meta");
        mMediaDb.execute("insert into meta values(0,0)");
    }

    public AnkiDb getMediaDb() {
        return mMediaDb;
    }

    /**
     * Remove media that is no longer being used from the SD-card.
     */
    public void removeUnusedImages() {
        List<String> listOfUnusedMedia = check().get(1); // Returns two lists, 2nd is unused media.
        for (String mediaName : listOfUnusedMedia) {
            File mediaFile = new File(mDir + "/" + mediaName);
            if (mediaFile.exists()) {
                mediaFile.delete();
            }
        }
    }

}