Java tutorial
package com.door43.translationstudio.core; import android.content.res.AssetManager; import com.door43.translationstudio.AppContext; import com.door43.translationstudio.rendering.USXtoUSFMConverter; import com.door43.util.FileUtilities; import com.door43.util.Manifest; import org.apache.commons.io.FileUtils; import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; import java.io.File; import java.io.FileFilter; import java.io.IOException; import java.io.InputStream; import java.util.Iterator; /** * Created by joel on 11/4/2015. */ public class TargetTranslationMigrator { private static final String MANIFEST_FILE = "manifest.json"; public static final String LICENSE = "LICENSE"; /** * Performs a migration on a manifest object. * We just throw it into a temporary directory and run the normal migration on it. * @param manifestJson * @return */ public static JSONObject migrateManifest(JSONObject manifestJson) { File tempDir = new File(AppContext.context().getCacheDir(), System.currentTimeMillis() + ""); // TRICKY: the migration can change the name of the translation dir so we nest it to avoid conflicts. File fakeTranslationDir = new File(tempDir, "translation"); fakeTranslationDir.mkdirs(); JSONObject migratedManifest = null; try { FileUtils.writeStringToFile(new File(fakeTranslationDir, "manifest.json"), manifestJson.toString()); fakeTranslationDir = migrate(fakeTranslationDir); if (fakeTranslationDir != null) { migratedManifest = new JSONObject( FileUtils.readFileToString(new File(fakeTranslationDir, "manifest.json"))); } } catch (Exception e) { e.printStackTrace(); } finally { // clean up FileUtils.deleteQuietly(tempDir); } return migratedManifest; } /** * Performs necessary migration operations on a target translation * @param targetTranslationDir * @return the target translation dir. Null if the migration failed */ public static File migrate(File targetTranslationDir) { File migratedDir = targetTranslationDir; File manifestFile = new File(targetTranslationDir, MANIFEST_FILE); try { JSONObject manifest = new JSONObject(FileUtils.readFileToString(manifestFile)); int packageVersion = 2; // default to version 2 if no package version is available if (manifest.has("package_version")) { packageVersion = manifest.getInt("package_version"); } switch (packageVersion) { case 2: migratedDir = v2(migratedDir); if (migratedDir == null) break; case 3: migratedDir = v3(migratedDir); if (migratedDir == null) break; case 4: migratedDir = v4(migratedDir); if (migratedDir == null) break; case 5: migratedDir = v5(migratedDir); if (migratedDir == null) break; case 6: migratedDir = v6(migratedDir); if (migratedDir == null) break; default: if (migratedDir != null && !validateTranslationType(migratedDir)) { migratedDir = null; } } } catch (Exception e) { e.printStackTrace(); migratedDir = null; } return migratedDir; } /** * current version * @param path * @return * @throws Exception */ private static File v6(File path) throws Exception { return path; } /** * Updated the id format of target translations * @param path * @return */ private static File v5(File path) throws Exception { File manifestFile = new File(path, MANIFEST_FILE); JSONObject manifest = new JSONObject(FileUtils.readFileToString(manifestFile)); // pull info to build id String targetLanguageCode = manifest.getJSONObject("target_language").getString("id"); String projectSlug = manifest.getJSONObject("project").getString("id"); String translationTypeSlug = manifest.getJSONObject("type").getString("id"); String resourceSlug = null; if (translationTypeSlug.equals("text")) { resourceSlug = manifest.getJSONObject("resource").getString("id"); } // build new id String id = targetLanguageCode + "_" + projectSlug + "_" + translationTypeSlug; if (translationTypeSlug.equals("text") && resourceSlug != null) { id += "_" + resourceSlug; } // add license file File licenseFile = new File(path, "LICENSE.md"); if (!licenseFile.exists()) { AssetManager am = AppContext.context().getAssets(); InputStream is = am.open("LICENSE.md"); if (is != null) { FileUtils.copyInputStreamToFile(is, licenseFile); } else { throw new Exception("Failed to open the template license file"); } } // update package version manifest.put("package_version", 6); FileUtils.write(manifestFile, manifest.toString(2)); // update target translation dir name File newPath = new File(path.getParentFile(), id.toLowerCase()); FileUtilities.safeDelete(newPath); FileUtils.moveDirectory(path, newPath); return newPath; } /** * major restructuring of the manifest to provide better support for future front/back matter, drafts, rendering, * and resolves issues between desktop and android platforms. * @param path * @return */ private static File v4(File path) throws Exception { File manifestFile = new File(path, MANIFEST_FILE); JSONObject manifest = new JSONObject(FileUtils.readFileToString(manifestFile)); // type { String typeId = "text"; if (manifest.has("project")) { try { JSONObject projectJson = manifest.getJSONObject("project"); typeId = projectJson.getString("type"); projectJson.remove("type"); manifest.put("project", projectJson); } catch (JSONException e) { e.printStackTrace(); } } JSONObject typeJson = new JSONObject(); TranslationType translationType = TranslationType.get(typeId); typeJson.put("id", typeId); if (translationType != null) { typeJson.put("name", translationType.getName()); } else { typeJson.put("name", ""); } manifest.put("type", typeJson); } // update project // NOTE: this was actually in v3 but we missed it so we need to catch it here if (manifest.has("project_id")) { String projectId = manifest.getString("project_id"); manifest.remove("project_id"); JSONObject projectJson = new JSONObject(); projectJson.put("id", projectId); projectJson.put("name", projectId.toUpperCase()); // we don't know the full name at this point manifest.put("project", projectJson); } // update resource if (manifest.getJSONObject("type").getString("id").equals("text")) { if (manifest.has("resource_id")) { String resourceId = manifest.getString("resource_id"); manifest.remove("resource_id"); JSONObject resourceJson = new JSONObject(); // TRICKY: supported resource id's (or now types) are "reg", "obs", "ulb", and "udb". if (resourceId.equals("ulb")) { resourceJson.put("name", "Unlocked Literal Bible"); } else if (resourceId.equals("udb")) { resourceJson.put("name", "Unlocked Dynamic Bible"); } else if (resourceId.equals("obs")) { resourceJson.put("name", "Open Bible Stories"); } else { // everything else changes to "reg" resourceId = "reg"; resourceJson.put("name", "Regular"); } resourceJson.put("id", resourceId); manifest.put("resource", resourceJson); } else if (!manifest.has("resource")) { // add missing resource JSONObject resourceJson = new JSONObject(); JSONObject projectJson = manifest.getJSONObject("project"); JSONObject typeJson = manifest.getJSONObject("type"); if (typeJson.getString("id").equals("text")) { String resourceId = projectJson.getString("id"); if (resourceId.equals("obs")) { resourceJson.put("id", "obs"); resourceJson.put("name", "Open Bible Stories"); } else { // everything else changes to reg resourceJson.put("id", "reg"); resourceJson.put("name", "Regular"); } manifest.put("resource", resourceJson); } } } else { // non-text translation types do not have resources manifest.remove("resource_id"); manifest.remove("resource"); } // update source translations if (manifest.has("source_translations")) { JSONObject oldSourceTranslationsJson = manifest.getJSONObject("source_translations"); manifest.remove("source_translations"); JSONArray newSourceTranslationsJson = new JSONArray(); Iterator<String> keys = oldSourceTranslationsJson.keys(); while (keys.hasNext()) { try { String key = keys.next(); JSONObject oldObj = oldSourceTranslationsJson.getJSONObject(key); JSONObject sourceTranslation = new JSONObject(); String[] parts = key.split("-", 2); if (parts.length == 2) { String languageResourceId = parts[1]; String[] pieces = languageResourceId.split("-"); if (pieces.length > 0) { String resId = pieces[pieces.length - 1]; sourceTranslation.put("resource_id", resId); sourceTranslation.put("language_id", languageResourceId.substring(0, languageResourceId.length() - resId.length() - 1)); sourceTranslation.put("checking_level", oldObj.getString("checking_level")); sourceTranslation.put("date_modified", oldObj.getInt("date_modified")); sourceTranslation.put("version", oldObj.getString("version")); newSourceTranslationsJson.put(sourceTranslation); } } } catch (Exception e) { // don't fail migration just because a source translation was invalid e.printStackTrace(); } } manifest.put("source_translations", newSourceTranslationsJson); } // update parent draft if (manifest.has("parent_draft_resource_id")) { JSONObject draftStatus = new JSONObject(); draftStatus.put("resource_id", manifest.getString("parent_draft_resource_id")); draftStatus.put("checking_entity", ""); draftStatus.put("checking_level", ""); draftStatus.put("comments", "The parent draft is unknown"); draftStatus.put("contributors", ""); draftStatus.put("publish_date", ""); draftStatus.put("source_text", ""); draftStatus.put("source_text_version", ""); draftStatus.put("version", ""); manifest.put("parent_draft", draftStatus); manifest.remove("parent_draft_resource_id"); } // update finished chunks if (manifest.has("finished_frames")) { JSONArray finishedFrames = manifest.getJSONArray("finished_frames"); manifest.remove("finished_frames"); manifest.put("finished_chunks", finishedFrames); } // remove finished titles if (manifest.has("finished_titles")) { JSONArray finishedChunks = manifest.getJSONArray("finished_chunks"); JSONArray finishedTitles = manifest.getJSONArray("finished_titles"); manifest.remove("finished_titles"); for (int i = 0; i < finishedTitles.length(); i++) { String chapterId = finishedTitles.getString(i); finishedChunks.put(chapterId + "-title"); } manifest.put("finished_chunks", finishedChunks); } // remove finished references if (manifest.has("finished_references")) { JSONArray finishedChunks = manifest.getJSONArray("finished_chunks"); JSONArray finishedReferences = manifest.getJSONArray("finished_references"); manifest.remove("finished_references"); for (int i = 0; i < finishedReferences.length(); i++) { String chapterId = finishedReferences.getString(i); finishedChunks.put(chapterId + "-reference"); } manifest.put("finished_chunks", finishedChunks); } // remove project components // NOTE: this was never quite official, just in android if (manifest.has("finished_project_components")) { JSONArray finishedChunks = manifest.getJSONArray("finished_chunks"); JSONArray finishedProjectComponents = manifest.getJSONArray("finished_project_components"); manifest.remove("finished_project_components"); for (int i = 0; i < finishedProjectComponents.length(); i++) { String component = finishedProjectComponents.getString(i); finishedChunks.put("00-" + component); } manifest.put("finished_chunks", finishedChunks); } // add format if (!Manifest.valueExists(manifest, "format") || manifest.getString("format").equals("usx") || manifest.getString("format").equals("default")) { String typeId = manifest.getJSONObject("type").getString("id"); String projectId = manifest.getJSONObject("project").getString("id"); if (!typeId.equals("text") || projectId.equals("obs")) { manifest.put("format", "markdown"); } else { manifest.put("format", "usfm"); } } // update where project title is saved. File oldProjectTitle = new File(path, "title.txt"); File newProjectTitle = new File(path, "00/title.txt"); if (oldProjectTitle.exists()) { newProjectTitle.getParentFile().mkdirs(); FileUtils.moveFile(oldProjectTitle, newProjectTitle); } // update package version manifest.put("package_version", 5); FileUtils.write(manifestFile, manifest.toString(2)); // migrate usx to usfm String format = manifest.getString("format"); // TRICKY: we just added the new format field, anything marked as usfm may have residual usx and needs to be migrated if (format.equals("usfm")) { File[] chapterDirs = path.listFiles(new FileFilter() { @Override public boolean accept(File pathname) { return pathname.isDirectory() && !pathname.getName().equals(".git"); } }); for (File cDir : chapterDirs) { File[] chunkFiles = cDir.listFiles(); for (File chunkFile : chunkFiles) { try { String usx = FileUtils.readFileToString(chunkFile); String usfm = USXtoUSFMConverter.doConversion(usx).toString(); FileUtils.writeStringToFile(chunkFile, usfm); } catch (IOException e) { // this conversion may have failed but don't stop the rest of the migration e.printStackTrace(); } } } } return path; } /** * We changed how the translator information is stored * we no longer store sensitive information like email and phone number * @param path * @return */ private static File v3(File path) throws Exception { File manifestFile = new File(path, MANIFEST_FILE); JSONObject manifest = new JSONObject(FileUtils.readFileToString(manifestFile)); if (manifest.has("translators")) { JSONArray legacyTranslators = manifest.getJSONArray("translators"); JSONArray translators = new JSONArray(); for (int i = 0; i < legacyTranslators.length(); i++) { Object obj = legacyTranslators.get(i); if (obj instanceof JSONObject) { translators.put(((JSONObject) obj).getString("name")); } else if (obj instanceof String) { translators.put(obj); } } manifest.put("translators", translators); manifest.put("package_version", 4); FileUtils.write(manifestFile, manifest.toString(2)); } migrateChunkChanges(path); return path; } /** * upgrade from v2 * @param path * @return */ private static File v2(File path) throws Exception { File manifestFile = new File(path, MANIFEST_FILE); JSONObject manifest = new JSONObject(FileUtils.readFileToString(manifestFile)); // fix finished frames if (manifest.has("frames")) { JSONObject legacyFrames = manifest.getJSONObject("frames"); Iterator<String> keys = legacyFrames.keys(); JSONArray finishedFrames = new JSONArray(); while (keys.hasNext()) { String key = keys.next(); JSONObject frameState = legacyFrames.getJSONObject(key); boolean finished = false; if (frameState.has("finished")) { finished = frameState.getBoolean("finished"); } if (finished) { finishedFrames.put(key); } } manifest.remove("frames"); manifest.put("finished_frames", finishedFrames); } // fix finished chapter titles and references if (manifest.has("chapters")) { JSONObject legacyChapters = manifest.getJSONObject("chapters"); Iterator<String> keys = legacyChapters.keys(); JSONArray finishedTitles = new JSONArray(); JSONArray finishedReferences = new JSONArray(); while (keys.hasNext()) { String key = keys.next(); JSONObject chapterState = legacyChapters.getJSONObject(key); boolean finishedTitle = false; boolean finishedReference = false; if (chapterState.has("finished_title")) { finishedTitle = chapterState.getBoolean("finished_title"); } if (chapterState.has("finished_reference")) { finishedTitle = chapterState.getBoolean("finished_reference"); } if (finishedTitle) { finishedTitles.put(key); } if (finishedReference) { finishedReferences.put(key); } } manifest.remove("chapters"); manifest.put("finished_titles", finishedTitles); manifest.put("finished_references", finishedReferences); } // fix project id if (manifest.has("slug")) { String projectSlug = manifest.getString("slug"); manifest.remove("slug"); manifest.put("project_id", projectSlug); } // fix target language id JSONObject targetLanguage = manifest.getJSONObject("target_language"); if (targetLanguage.has("slug")) { String targetLanguageSlug = targetLanguage.getString("slug"); targetLanguage.remove("slug"); targetLanguage.put("id", targetLanguageSlug); manifest.put("target_language", targetLanguage); } manifest.put("package_version", 3); FileUtils.write(manifestFile, manifest.toString(2)); return path; } /** * Merges chunks found in a target translation Project that do not exist in the source translation * to a sibling chunk so that no data is lost. * @param targetTranslationDir * @return */ private static boolean migrateChunkChanges(File targetTranslationDir) { // TRICKY: calling the AppContext here is bad practice, but we'll deprecate this soon anyway. final Library library = AppContext.getLibrary(); final SourceTranslation sourceTranslation = library .getDefaultSourceTranslation(targetTranslationDir.getName(), "en"); if (sourceTranslation == null) { // if there is no source we are done return true; } File[] chapterDirs = targetTranslationDir.listFiles(new FileFilter() { @Override public boolean accept(File pathname) { return pathname.isDirectory() && !pathname.getName().equals(".git") && !pathname.getName().equals("00"); // 00 contains project title translations } }); for (File cDir : chapterDirs) { mergeInvalidChunksInChapter(library, new File(targetTranslationDir, "manifest.json"), sourceTranslation, cDir); } return true; } /** * Merges invalid chunks found in the target translation with a valid sibling chunk in order * to preserve translation data. Merged chunks are marked as not finished to force * translators to review the changes. * @param library * @param manifestFile * @param sourceTranslation * @param chapterDir * @return */ private static boolean mergeInvalidChunksInChapter(final Library library, File manifestFile, final SourceTranslation sourceTranslation, final File chapterDir) { JSONObject manifest; try { manifest = new JSONObject(FileUtils.readFileToString(manifestFile)); } catch (Exception e) { e.printStackTrace(); return false; } final String chunkMergeMarker = "\n----------\n"; File[] frameFiles = chapterDir.listFiles(new FileFilter() { @Override public boolean accept(File pathname) { return !pathname.getName().equals("title.txt") && !pathname.getName().equals("reference.txt"); } }); String invalidChunks = ""; File lastValidFrameFile = null; String chapterId = chapterDir.getName(); for (File frameFile : frameFiles) { String frameId = frameFile.getName(); Frame frame = library.getFrame(sourceTranslation, chapterId, frameId); String frameBody = ""; try { frameBody = FileUtils.readFileToString(frameFile).trim(); } catch (Exception e) { e.printStackTrace(); } if (frame != null) { lastValidFrameFile = frameFile; // merge invalid frames into the existing frame if (!invalidChunks.isEmpty()) { try { FileUtils.writeStringToFile(frameFile, invalidChunks + frameBody); } catch (IOException e) { e.printStackTrace(); } invalidChunks = ""; try { Manifest.removeValue(manifest.getJSONArray("finished_frames"), chapterId + "-" + frameId); } catch (JSONException e) { e.printStackTrace(); } } } else if (!frameBody.isEmpty()) { // collect invalid frame if (lastValidFrameFile == null) { invalidChunks += frameBody + chunkMergeMarker; } else { // append to last valid frame String lastValidFrameBody = ""; try { lastValidFrameBody = FileUtils.readFileToString(lastValidFrameFile); } catch (IOException e) { e.printStackTrace(); } try { FileUtils.writeStringToFile(lastValidFrameFile, lastValidFrameBody + chunkMergeMarker + frameBody); } catch (IOException e) { e.printStackTrace(); } try { Manifest.removeValue(manifest.getJSONArray("finished_frames"), chapterId + "-" + lastValidFrameFile.getName()); } catch (JSONException e) { e.printStackTrace(); } } // delete invalid frame FileUtils.deleteQuietly(frameFile); } } // clean up remaining invalid chunks if (!invalidChunks.isEmpty()) { // grab updated list of frames frameFiles = chapterDir.listFiles(new FileFilter() { @Override public boolean accept(File pathname) { return !pathname.getName().equals("title.txt") && !pathname.getName().equals("reference.txt"); } }); if (frameFiles != null && frameFiles.length > 0) { String frameBody = ""; try { frameBody = FileUtils.readFileToString(frameFiles[0]); } catch (IOException e) { e.printStackTrace(); } try { FileUtils.writeStringToFile(frameFiles[0], invalidChunks + chunkMergeMarker + frameBody); try { Manifest.removeValue(manifest.getJSONArray("finished_frames"), chapterId + "-" + frameFiles[0].getName()); } catch (JSONException e) { e.printStackTrace(); } } catch (IOException e) { e.printStackTrace(); } } } return true; } /** * Checks if the android app can support this translation type. * Example: ts-desktop can translate tW but ts-android cannot. * @param path * @return */ private static boolean validateTranslationType(File path) throws Exception { JSONObject manifest = new JSONObject(FileUtils.readFileToString(new File(path, MANIFEST_FILE))); String typeId = manifest.getJSONObject("type").getString("id"); // android only supports TEXT translations for now return TranslationType.get(typeId) == TranslationType.TEXT; } }