List of usage examples for java.util ArrayList removeAll
public boolean removeAll(Collection<?> c)
From source file:org.apache.sysml.lops.compile.Dag.java
/** * Method to generate MapReduce job instructions from a given set of nodes. * /*from www . j av a 2 s.c om*/ * @param execNodes list of exec nodes * @param inst list of instructions * @param writeinst list of write instructions * @param deleteinst list of delete instructions * @param rmvarinst list of rmvar instructions * @param jt job type * @throws LopsException if LopsException occurs * @throws DMLRuntimeException if DMLRuntimeException occurs */ private void generateMapReduceInstructions(ArrayList<Lop> execNodes, ArrayList<Instruction> inst, ArrayList<Instruction> writeinst, ArrayList<Instruction> deleteinst, ArrayList<Instruction> rmvarinst, JobType jt) throws LopsException, DMLRuntimeException { ArrayList<Byte> resultIndices = new ArrayList<Byte>(); ArrayList<String> inputs = new ArrayList<String>(); ArrayList<String> outputs = new ArrayList<String>(); ArrayList<InputInfo> inputInfos = new ArrayList<InputInfo>(); ArrayList<OutputInfo> outputInfos = new ArrayList<OutputInfo>(); ArrayList<Long> numRows = new ArrayList<Long>(); ArrayList<Long> numCols = new ArrayList<Long>(); ArrayList<Long> numRowsPerBlock = new ArrayList<Long>(); ArrayList<Long> numColsPerBlock = new ArrayList<Long>(); ArrayList<String> mapperInstructions = new ArrayList<String>(); ArrayList<String> randInstructions = new ArrayList<String>(); ArrayList<String> recordReaderInstructions = new ArrayList<String>(); int numReducers = 0; int replication = 1; ArrayList<String> inputLabels = new ArrayList<String>(); ArrayList<String> outputLabels = new ArrayList<String>(); ArrayList<Instruction> renameInstructions = new ArrayList<Instruction>(); ArrayList<Instruction> variableInstructions = new ArrayList<Instruction>(); ArrayList<Instruction> postInstructions = new ArrayList<Instruction>(); ArrayList<Integer> MRJobLineNumbers = null; if (DMLScript.ENABLE_DEBUG_MODE) { MRJobLineNumbers = new ArrayList<Integer>(); } ArrayList<Lop> inputLops = new ArrayList<Lop>(); boolean cellModeOverride = false; /* Find the nodes that produce an output */ ArrayList<Lop> rootNodes = new ArrayList<Lop>(); getOutputNodes(execNodes, rootNodes, jt); if (LOG.isTraceEnabled()) LOG.trace("# of root nodes = " + rootNodes.size()); /* Remove transient writes that are simple copy of transient reads */ if (jt == JobType.GMR || jt == JobType.GMRCELL) { ArrayList<Lop> markedNodes = new ArrayList<Lop>(); // only keep data nodes that are results of some computation. for (Lop rnode : rootNodes) { if (rnode.getExecLocation() == ExecLocation.Data && ((Data) rnode).isTransient() && ((Data) rnode).getOperationType() == OperationTypes.WRITE && ((Data) rnode).getDataType() == DataType.MATRIX) { // no computation, just a copy if (rnode.getInputs().get(0).getExecLocation() == ExecLocation.Data && ((Data) rnode.getInputs().get(0)).isTransient() && rnode.getOutputParameters() .getLabel().equals(rnode.getInputs().get(0).getOutputParameters().getLabel())) { markedNodes.add(rnode); } } } // delete marked nodes rootNodes.removeAll(markedNodes); markedNodes.clear(); if (rootNodes.isEmpty()) return; } // structure that maps node to their indices that will be used in the instructions HashMap<Lop, Integer> nodeIndexMapping = new HashMap<Lop, Integer>(); /* Determine all input data files */ for (Lop rnode : rootNodes) { getInputPathsAndParameters(rnode, execNodes, inputs, inputInfos, numRows, numCols, numRowsPerBlock, numColsPerBlock, nodeIndexMapping, inputLabels, inputLops, MRJobLineNumbers); } // In case of RAND job, instructions are defined in the input file if (jt == JobType.DATAGEN) randInstructions = inputs; int[] start_index = new int[1]; start_index[0] = inputs.size(); /* Get RecordReader Instructions */ // currently, recordreader instructions are allowed only in GMR jobs if (jt == JobType.GMR || jt == JobType.GMRCELL) { for (Lop rnode : rootNodes) { getRecordReaderInstructions(rnode, execNodes, inputs, recordReaderInstructions, nodeIndexMapping, start_index, inputLabels, inputLops, MRJobLineNumbers); if (recordReaderInstructions.size() > 1) throw new LopsException("MapReduce job can only have a single recordreader instruction: " + recordReaderInstructions.toString()); } } /* * Handle cases when job's output is FORCED to be cell format. * - If there exist a cell input, then output can not be blocked. * Only exception is when jobType = REBLOCK/CSVREBLOCK (for obvisous reason) * or when jobType = RAND since RandJob takes a special input file, * whose format should not be used to dictate the output format. * - If there exists a recordReader instruction * - If jobtype = GroupedAgg. This job can only run in cell mode. */ // if (jt != JobType.REBLOCK && jt != JobType.CSV_REBLOCK && jt != JobType.DATAGEN && jt != JobType.TRANSFORM) { for (int i = 0; i < inputInfos.size(); i++) if (inputInfos.get(i) == InputInfo.BinaryCellInputInfo || inputInfos.get(i) == InputInfo.TextCellInputInfo) cellModeOverride = true; } if (!recordReaderInstructions.isEmpty() || jt == JobType.GROUPED_AGG) cellModeOverride = true; /* Get Mapper Instructions */ for (int i = 0; i < rootNodes.size(); i++) { getMapperInstructions(rootNodes.get(i), execNodes, inputs, mapperInstructions, nodeIndexMapping, start_index, inputLabels, inputLops, MRJobLineNumbers); } if (LOG.isTraceEnabled()) { LOG.trace(" Input strings: " + inputs.toString()); if (jt == JobType.DATAGEN) LOG.trace(" Rand instructions: " + getCSVString(randInstructions)); if (jt == JobType.GMR) LOG.trace(" RecordReader instructions: " + getCSVString(recordReaderInstructions)); LOG.trace(" Mapper instructions: " + getCSVString(mapperInstructions)); } /* Get Shuffle and Reducer Instructions */ ArrayList<String> shuffleInstructions = new ArrayList<String>(); ArrayList<String> aggInstructionsReducer = new ArrayList<String>(); ArrayList<String> otherInstructionsReducer = new ArrayList<String>(); for (Lop rn : rootNodes) { int resultIndex = getAggAndOtherInstructions(rn, execNodes, shuffleInstructions, aggInstructionsReducer, otherInstructionsReducer, nodeIndexMapping, start_index, inputLabels, inputLops, MRJobLineNumbers); if (resultIndex == -1) throw new LopsException("Unexpected error in piggybacking!"); if (rn.getExecLocation() == ExecLocation.Data && ((Data) rn).getOperationType() == Data.OperationTypes.WRITE && ((Data) rn).isTransient() && rootNodes.contains(rn.getInputs().get(0))) { // Both rn (a transient write) and its input are root nodes. // Instead of creating two copies of the data, simply generate a cpvar instruction NodeOutput out = setupNodeOutputs(rn, ExecType.MR, cellModeOverride, true); writeinst.addAll(out.getLastInstructions()); } else { resultIndices.add(Byte.valueOf((byte) resultIndex)); // setup output filenames and outputInfos and generate related instructions NodeOutput out = setupNodeOutputs(rn, ExecType.MR, cellModeOverride, false); outputLabels.add(out.getVarName()); outputs.add(out.getFileName()); outputInfos.add(out.getOutInfo()); if (LOG.isTraceEnabled()) { LOG.trace(" Output Info: " + out.getFileName() + ";" + OutputInfo.outputInfoToString(out.getOutInfo()) + ";" + out.getVarName()); } renameInstructions.addAll(out.getLastInstructions()); variableInstructions.addAll(out.getPreInstructions()); postInstructions.addAll(out.getPostInstructions()); } } /* Determine if the output dimensions are known */ byte[] resultIndicesByte = new byte[resultIndices.size()]; for (int i = 0; i < resultIndicesByte.length; i++) { resultIndicesByte[i] = resultIndices.get(i).byteValue(); } if (LOG.isTraceEnabled()) { LOG.trace(" Shuffle Instructions: " + getCSVString(shuffleInstructions)); LOG.trace(" Aggregate Instructions: " + getCSVString(aggInstructionsReducer)); LOG.trace(" Other instructions =" + getCSVString(otherInstructionsReducer)); LOG.trace(" Output strings: " + outputs.toString()); LOG.trace(" ResultIndices = " + resultIndices.toString()); } /* Prepare the MapReduce job instruction */ MRJobInstruction mr = new MRJobInstruction(jt); // check if this is a map-only job. If not, set the number of reducers if (!shuffleInstructions.isEmpty() || !aggInstructionsReducer.isEmpty() || !otherInstructionsReducer.isEmpty()) numReducers = total_reducers; // set inputs, outputs, and other other properties for the job mr.setInputOutputLabels(inputLabels.toArray(new String[0]), outputLabels.toArray(new String[0])); mr.setOutputs(resultIndicesByte); mr.setDimsUnknownFilePrefix(getFilePath()); mr.setNumberOfReducers(numReducers); mr.setReplication(replication); // set instructions for recordReader and mapper mr.setRecordReaderInstructions(getCSVString(recordReaderInstructions)); mr.setMapperInstructions(getCSVString(mapperInstructions)); //compute and set mapper memory requirements (for consistency of runtime piggybacking) if (jt == JobType.GMR) { double mem = 0; for (Lop n : execNodes) mem += computeFootprintInMapper(n); mr.setMemoryRequirements(mem); } if (jt == JobType.DATAGEN) mr.setRandInstructions(getCSVString(randInstructions)); // set shuffle instructions mr.setShuffleInstructions(getCSVString(shuffleInstructions)); // set reducer instruction mr.setAggregateInstructionsInReducer(getCSVString(aggInstructionsReducer)); mr.setOtherInstructionsInReducer(getCSVString(otherInstructionsReducer)); if (DMLScript.ENABLE_DEBUG_MODE) { // set line number information for each MR instruction mr.setMRJobInstructionsLineNumbers(MRJobLineNumbers); } /* Add the prepared instructions to output set */ inst.addAll(variableInstructions); inst.add(mr); inst.addAll(postInstructions); deleteinst.addAll(renameInstructions); for (Lop l : inputLops) { if (DMLScript.ENABLE_DEBUG_MODE) { processConsumers(l, rmvarinst, deleteinst, l); } else { processConsumers(l, rmvarinst, deleteinst, null); } } }
From source file:com.ibm.bi.dml.lops.compile.Dag.java
/** * Method to generate MapReduce job instructions from a given set of nodes. * /* w w w.java 2 s.co m*/ * @param execNodes * @param inst * @param deleteinst * @param jobType * @throws LopsException * @throws DMLUnsupportedOperationException * @throws DMLRuntimeException */ @SuppressWarnings("unchecked") public void generateMapReduceInstructions(ArrayList<N> execNodes, ArrayList<Instruction> inst, ArrayList<Instruction> writeinst, ArrayList<Instruction> deleteinst, ArrayList<Instruction> rmvarinst, JobType jt) throws LopsException, DMLUnsupportedOperationException, DMLRuntimeException { ArrayList<Byte> resultIndices = new ArrayList<Byte>(); ArrayList<String> inputs = new ArrayList<String>(); ArrayList<String> outputs = new ArrayList<String>(); ArrayList<InputInfo> inputInfos = new ArrayList<InputInfo>(); ArrayList<OutputInfo> outputInfos = new ArrayList<OutputInfo>(); ArrayList<Long> numRows = new ArrayList<Long>(); ArrayList<Long> numCols = new ArrayList<Long>(); ArrayList<Long> numRowsPerBlock = new ArrayList<Long>(); ArrayList<Long> numColsPerBlock = new ArrayList<Long>(); ArrayList<String> mapperInstructions = new ArrayList<String>(); ArrayList<String> randInstructions = new ArrayList<String>(); ArrayList<String> recordReaderInstructions = new ArrayList<String>(); int numReducers = 0; int replication = 1; ArrayList<String> inputLabels = new ArrayList<String>(); ArrayList<String> outputLabels = new ArrayList<String>(); ArrayList<Instruction> renameInstructions = new ArrayList<Instruction>(); ArrayList<Instruction> variableInstructions = new ArrayList<Instruction>(); ArrayList<Instruction> postInstructions = new ArrayList<Instruction>(); ArrayList<Integer> MRJobLineNumbers = null; if (DMLScript.ENABLE_DEBUG_MODE) { MRJobLineNumbers = new ArrayList<Integer>(); } ArrayList<Lop> inputLops = new ArrayList<Lop>(); boolean cellModeOverride = false; /* Find the nodes that produce an output */ ArrayList<N> rootNodes = new ArrayList<N>(); getOutputNodes(execNodes, rootNodes, jt); if (LOG.isTraceEnabled()) LOG.trace("# of root nodes = " + rootNodes.size()); /* Remove transient writes that are simple copy of transient reads */ if (jt == JobType.GMR || jt == JobType.GMRCELL) { ArrayList<N> markedNodes = new ArrayList<N>(); // only keep data nodes that are results of some computation. for (int i = 0; i < rootNodes.size(); i++) { N node = rootNodes.get(i); if (node.getExecLocation() == ExecLocation.Data && ((Data) node).isTransient() && ((Data) node).getOperationType() == OperationTypes.WRITE && ((Data) node).getDataType() == DataType.MATRIX) { // no computation, just a copy if (node.getInputs().get(0).getExecLocation() == ExecLocation.Data && ((Data) node.getInputs().get(0)).isTransient() && node.getOutputParameters().getLabel() .compareTo(node.getInputs().get(0).getOutputParameters().getLabel()) == 0) { markedNodes.add(node); } } } // delete marked nodes rootNodes.removeAll(markedNodes); markedNodes.clear(); if (rootNodes.isEmpty()) return; } // structure that maps node to their indices that will be used in the instructions HashMap<N, Integer> nodeIndexMapping = new HashMap<N, Integer>(); /* Determine all input data files */ for (int i = 0; i < rootNodes.size(); i++) { getInputPathsAndParameters(rootNodes.get(i), execNodes, inputs, inputInfos, numRows, numCols, numRowsPerBlock, numColsPerBlock, nodeIndexMapping, inputLabels, inputLops, MRJobLineNumbers); } // In case of RAND job, instructions are defined in the input file if (jt == JobType.DATAGEN) randInstructions = inputs; int[] start_index = new int[1]; start_index[0] = inputs.size(); /* Get RecordReader Instructions */ // currently, recordreader instructions are allowed only in GMR jobs if (jt == JobType.GMR || jt == JobType.GMRCELL) { for (int i = 0; i < rootNodes.size(); i++) { getRecordReaderInstructions(rootNodes.get(i), execNodes, inputs, recordReaderInstructions, nodeIndexMapping, start_index, inputLabels, inputLops, MRJobLineNumbers); if (recordReaderInstructions.size() > 1) throw new LopsException("MapReduce job can only have a single recordreader instruction: " + recordReaderInstructions.toString()); } } /* * Handle cases when job's output is FORCED to be cell format. * - If there exist a cell input, then output can not be blocked. * Only exception is when jobType = REBLOCK/CSVREBLOCK (for obvisous reason) * or when jobType = RAND since RandJob takes a special input file, * whose format should not be used to dictate the output format. * - If there exists a recordReader instruction * - If jobtype = GroupedAgg. This job can only run in cell mode. */ // if (jt != JobType.REBLOCK && jt != JobType.CSV_REBLOCK && jt != JobType.DATAGEN && jt != JobType.TRANSFORM) { for (int i = 0; i < inputInfos.size(); i++) if (inputInfos.get(i) == InputInfo.BinaryCellInputInfo || inputInfos.get(i) == InputInfo.TextCellInputInfo) cellModeOverride = true; } if (!recordReaderInstructions.isEmpty() || jt == JobType.GROUPED_AGG) cellModeOverride = true; /* Get Mapper Instructions */ for (int i = 0; i < rootNodes.size(); i++) { getMapperInstructions(rootNodes.get(i), execNodes, inputs, mapperInstructions, nodeIndexMapping, start_index, inputLabels, inputLops, MRJobLineNumbers); } if (LOG.isTraceEnabled()) { LOG.trace(" Input strings: " + inputs.toString()); if (jt == JobType.DATAGEN) LOG.trace(" Rand instructions: " + getCSVString(randInstructions)); if (jt == JobType.GMR) LOG.trace(" RecordReader instructions: " + getCSVString(recordReaderInstructions)); LOG.trace(" Mapper instructions: " + getCSVString(mapperInstructions)); } /* Get Shuffle and Reducer Instructions */ ArrayList<String> shuffleInstructions = new ArrayList<String>(); ArrayList<String> aggInstructionsReducer = new ArrayList<String>(); ArrayList<String> otherInstructionsReducer = new ArrayList<String>(); for (int i = 0; i < rootNodes.size(); i++) { N rn = rootNodes.get(i); int resultIndex = getAggAndOtherInstructions(rn, execNodes, shuffleInstructions, aggInstructionsReducer, otherInstructionsReducer, nodeIndexMapping, start_index, inputLabels, inputLops, MRJobLineNumbers); if (resultIndex == -1) throw new LopsException("Unexpected error in piggybacking!"); if (rn.getExecLocation() == ExecLocation.Data && ((Data) rn).getOperationType() == Data.OperationTypes.WRITE && ((Data) rn).isTransient() && rootNodes.contains(rn.getInputs().get(0))) { // Both rn (a transient write) and its input are root nodes. // Instead of creating two copies of the data, simply generate a cpvar instruction NodeOutput out = setupNodeOutputs(rootNodes.get(i), ExecType.MR, cellModeOverride, true); writeinst.addAll(out.getLastInstructions()); } else { resultIndices.add(Byte.valueOf((byte) resultIndex)); // setup output filenames and outputInfos and generate related instructions NodeOutput out = setupNodeOutputs(rootNodes.get(i), ExecType.MR, cellModeOverride, false); outputLabels.add(out.getVarName()); outputs.add(out.getFileName()); outputInfos.add(out.getOutInfo()); if (LOG.isTraceEnabled()) { LOG.trace(" Output Info: " + out.getFileName() + ";" + OutputInfo.outputInfoToString(out.getOutInfo()) + ";" + out.getVarName()); } renameInstructions.addAll(out.getLastInstructions()); variableInstructions.addAll(out.getPreInstructions()); postInstructions.addAll(out.getPostInstructions()); } } /* Determine if the output dimensions are known */ byte[] resultIndicesByte = new byte[resultIndices.size()]; for (int i = 0; i < resultIndicesByte.length; i++) { resultIndicesByte[i] = resultIndices.get(i).byteValue(); } if (LOG.isTraceEnabled()) { LOG.trace(" Shuffle Instructions: " + getCSVString(shuffleInstructions)); LOG.trace(" Aggregate Instructions: " + getCSVString(aggInstructionsReducer)); LOG.trace(" Other instructions =" + getCSVString(otherInstructionsReducer)); LOG.trace(" Output strings: " + outputs.toString()); LOG.trace(" ResultIndices = " + resultIndices.toString()); } /* Prepare the MapReduce job instruction */ MRJobInstruction mr = new MRJobInstruction(jt); // check if this is a map-only job. If not, set the number of reducers if (!shuffleInstructions.isEmpty() || !aggInstructionsReducer.isEmpty() || !otherInstructionsReducer.isEmpty()) numReducers = total_reducers; // set inputs, outputs, and other other properties for the job mr.setInputOutputLabels(getStringArray(inputLabels), getStringArray(outputLabels)); mr.setOutputs(resultIndicesByte); mr.setDimsUnknownFilePrefix(getFilePath()); mr.setNumberOfReducers(numReducers); mr.setReplication(replication); // set instructions for recordReader and mapper mr.setRecordReaderInstructions(getCSVString(recordReaderInstructions)); mr.setMapperInstructions(getCSVString(mapperInstructions)); //compute and set mapper memory requirements (for consistency of runtime piggybacking) if (jt == JobType.GMR) { double mem = 0; for (N n : execNodes) mem += computeFootprintInMapper(n); mr.setMemoryRequirements(mem); } if (jt == JobType.DATAGEN) mr.setRandInstructions(getCSVString(randInstructions)); // set shuffle instructions mr.setShuffleInstructions(getCSVString(shuffleInstructions)); // set reducer instruction mr.setAggregateInstructionsInReducer(getCSVString(aggInstructionsReducer)); mr.setOtherInstructionsInReducer(getCSVString(otherInstructionsReducer)); if (DMLScript.ENABLE_DEBUG_MODE) { // set line number information for each MR instruction mr.setMRJobInstructionsLineNumbers(MRJobLineNumbers); } /* Add the prepared instructions to output set */ inst.addAll(variableInstructions); inst.add(mr); inst.addAll(postInstructions); deleteinst.addAll(renameInstructions); for (Lop l : inputLops) { if (DMLScript.ENABLE_DEBUG_MODE) { processConsumers((N) l, rmvarinst, deleteinst, (N) l); } else { processConsumers((N) l, rmvarinst, deleteinst, null); } } }
From source file:org.tinymediamanager.core.movie.MovieRenamer.java
/** * Rename movie./*from w w w .j av a 2s . co m*/ * * @param movie * the movie */ public static void renameMovie(Movie movie) { // FIXME: what? when? boolean posterRenamed = false; boolean fanartRenamed = false; boolean downloadMissingArtworks = false; // check if a datasource is set if (StringUtils.isEmpty(movie.getDataSource())) { LOGGER.error("no Datasource set"); return; } // if (!movie.isScraped()) { if (movie.getTitle().isEmpty()) { LOGGER.error("won't rename movie '" + movie.getPathNIO() + "' / '" + movie.getTitle() + "' not even title is set?"); return; } // all the good & needed mediafiles ArrayList<MediaFile> needed = new ArrayList<>(); ArrayList<MediaFile> cleanup = new ArrayList<>(); LOGGER.info("Renaming movie: " + movie.getTitle()); LOGGER.debug("movie year: " + movie.getYear()); LOGGER.debug("movie path: " + movie.getPathNIO()); LOGGER.debug("movie isDisc?: " + movie.isDisc()); LOGGER.debug("movie isMulti?: " + movie.isMultiMovieDir()); if (movie.getMovieSet() != null) { LOGGER.debug("movieset: " + movie.getMovieSet().getTitle()); } LOGGER.debug("path expression: " + MovieModuleManager.MOVIE_SETTINGS.getMovieRenamerPathname()); LOGGER.debug("file expression: " + MovieModuleManager.MOVIE_SETTINGS.getMovieRenamerFilename()); String newPathname = createDestinationForFoldername( MovieModuleManager.MOVIE_SETTINGS.getMovieRenamerPathname(), movie); String oldPathname = movie.getPathNIO().toString(); if (!newPathname.isEmpty()) { newPathname = movie.getDataSource() + File.separator + newPathname; Path srcDir = movie.getPathNIO(); Path destDir = Paths.get(newPathname); if (!srcDir.toAbsolutePath().equals(destDir.toAbsolutePath())) { boolean newDestIsMultiMovieDir = false; // re-evaluate multiMovieDir based on renamer settings // folder MUST BE UNIQUE, we need at least a T/E-Y combo or IMDBid // so if renaming just to a fixed pattern (eg "$S"), movie will downgrade to a MMD if (!isFolderPatternUnique(MovieModuleManager.MOVIE_SETTINGS.getMovieRenamerPathname())) { // FIXME: if we already in a normal dir - keep it? newDestIsMultiMovieDir = true; } // FIXME: add warning to GUI if downgrade!!!!!! LOGGER.debug("movie willBeMulti?: " + newDestIsMultiMovieDir); // ###################################################################### // ## 1) old = separate movie dir, and new too -> move folder // ###################################################################### if (!movie.isMultiMovieDir() && !newDestIsMultiMovieDir) { boolean ok = false; try { ok = Utils.moveDirectorySafe(srcDir, destDir); if (ok) { movie.setMultiMovieDir(false); movie.updateMediaFilePath(srcDir, destDir); movie.setPath(newPathname); movie.saveToDb(); // since we moved already, save it } } catch (Exception e) { LOGGER.error("error moving folder: ", e); MessageManager.instance.pushMessage(new Message(MessageLevel.ERROR, srcDir, "message.renamer.failedrename", new String[] { ":", e.getLocalizedMessage() })); } if (!ok) { // FIXME: when we were not able to rename folder, display error msg and abort!!! LOGGER.error("Could not move to destination '" + destDir + "' - NOT renaming folder"); return; } } else if (movie.isMultiMovieDir() && !newDestIsMultiMovieDir) { // ###################################################################### // ## 2) MMD movie -> normal movie (upgrade) // ###################################################################### LOGGER.trace("Upgrading movie into it's own dir :) " + newPathname); try { Files.createDirectories(destDir); } catch (Exception e) { LOGGER.error("Could not create destination '" + destDir + "' - NOT renaming folder ('upgrade' movie)"); // well, better not to rename return; } movie.setMultiMovieDir(false); downloadMissingArtworks = true; // yay - we upgraded our movie, so we could try to get additional artworks :) } else { // ###################################################################### // ## Can be // ## 3) MMD movie -> MMD movie (but foldername possible changed) // ## 4) normal movie -> MMD movie (downgrade) // ## either way - check & create dest folder // ###################################################################### LOGGER.trace("New movie path is a MMD :( " + newPathname); if (!Files.exists(destDir)) { // if existent, all is good -> MMD (FIXME: kinda, we *might* have another full movie in there) try { Files.createDirectories(destDir); } catch (Exception e) { LOGGER.error("Could not create destination '" + destDir + "' - NOT renaming folder ('MMD' movie)"); // well, better not to rename return; } } movie.setMultiMovieDir(true); } } // src == dest } // folder pattern empty else { LOGGER.info("Folder rename settings were empty - NOT renaming folder"); // set it to current for file renaming newPathname = movie.getPathNIO().toString(); } // ###################################################################### // ## mark ALL existing and known files for cleanup (clone!!) // ###################################################################### for (MovieNfoNaming s : MovieNfoNaming.values()) { String nfoFilename = movie.getNfoFilename(s); if (StringUtils.isBlank(nfoFilename)) { continue; } // mark all known variants for cleanup MediaFile del = new MediaFile(movie.getPathNIO().resolve(nfoFilename), MediaFileType.NFO); cleanup.add(del); } for (MoviePosterNaming s : MoviePosterNaming.values()) { MediaFile del = new MediaFile( movie.getPathNIO() .resolve(replaceInvalidCharacters(MovieArtworkHelper.getPosterFilename(s, movie))), MediaFileType.POSTER); cleanup.add(del); } for (MovieFanartNaming s : MovieFanartNaming.values()) { MediaFile del = new MediaFile( movie.getPathNIO() .resolve(replaceInvalidCharacters(MovieArtworkHelper.getFanartFilename(s, movie))), MediaFileType.FANART); cleanup.add(del); } // cleanup ALL MFs for (MediaFile del : movie.getMediaFiles()) { cleanup.add(new MediaFile(del)); } cleanup.removeAll(Collections.singleton(null)); // remove all NULL ones! // update movie path at end of renaming - we need the old one here!! // movie.setPath(newPathname); // movie.saveToDb(); // BASENAME String newVideoBasename = ""; if (!isFilePatternValid()) { // Template empty or not even title set, so we are NOT renaming any files // we keep the same name on renaming ;) newVideoBasename = movie.getVideoBasenameWithoutStacking(); LOGGER.warn("Filepattern is not valid - NOT renaming files!"); } else { // since we rename, generate the new basename MediaFile ftr = generateFilename(movie, movie.getMediaFiles(MediaFileType.VIDEO).get(0), newVideoBasename).get(0); // there can be only one newVideoBasename = FilenameUtils.getBaseName(ftr.getFilenameWithoutStacking()); } LOGGER.debug("Our new basename for renaming: " + newVideoBasename); // unneeded / more reliable with with java 7? // // ###################################################################### // // ## test VIDEO rename // // ###################################################################### // for (MediaFile vid : movie.getMediaFiles(MediaFileType.VIDEO)) { // LOGGER.debug("testing file " + vid.getFileAsPath()); // Path f = vid.getFileAsPath(); // boolean testRenameOk = false; // for (int i = 0; i < 5; i++) { // testRenameOk = f.renameTo(f); // haahaa, try to rename to itself :P // if (testRenameOk) { // break; // ok it worked, step out // } // // we had the case, that the renaemoTo didn't work, // // and even the exists did not work! // // so we skip this additional check, which results in not removing the movie file // // if (!f.exists()) { // // LOGGER.debug("Hmmm... file " + f + " does not even exists; delete from DB"); // // // delete from MF or ignore for later cleanup (but better now!) // // movie.removeFromMediaFiles(vid); // // testRenameOk = true; // we "tested" this ok // // break; // // } // try { // LOGGER.debug("rename did not work - sleep a while and try again..."); // Thread.sleep(1000); // } // catch (InterruptedException e) { // LOGGER.warn("I'm so excited - could not sleep"); // } // } // if (!testRenameOk) { // LOGGER.warn("File " + vid.getFileAsPath() + " is not accessible!"); // MessageManager.instance.pushMessage(new Message(MessageLevel.ERROR, vid.getFilename(), "message.renamer.failedrename")); // return; // } // } // ###################################################################### // ## rename VIDEO (move 1:1) // ###################################################################### for (MediaFile vid : movie.getMediaFiles(MediaFileType.VIDEO)) { LOGGER.trace("Rename 1:1 " + vid.getType() + " " + vid.getFileAsPath()); MediaFile newMF = generateFilename(movie, vid, newVideoBasename).get(0); // there can be only one boolean ok = moveFile(vid.getFileAsPath(), newMF.getFileAsPath()); if (ok) { vid.setFile(newMF.getFileAsPath()); // update } needed.add(vid); // add vid, since we're updating existing MF object } // ###################################################################### // ## rename POSTER, FANART (copy 1:N) // ###################################################################### // we can have multiple ones, just get the newest one and copy(overwrite) them to all needed ArrayList<MediaFile> mfs = new ArrayList<>(); mfs.add(movie.getNewestMediaFilesOfType(MediaFileType.FANART)); mfs.add(movie.getNewestMediaFilesOfType(MediaFileType.POSTER)); mfs.removeAll(Collections.singleton(null)); // remove all NULL ones! for (MediaFile mf : mfs) { LOGGER.trace("Rename 1:N " + mf.getType() + " " + mf.getFileAsPath()); ArrayList<MediaFile> newMFs = generateFilename(movie, mf, newVideoBasename); // 1:N for (MediaFile newMF : newMFs) { posterRenamed = true; fanartRenamed = true; boolean ok = copyFile(mf.getFileAsPath(), newMF.getFileAsPath()); if (ok) { needed.add(newMF); } } } // ###################################################################### // ## rename NFO (copy 1:N) - only TMM NFOs // ###################################################################### // we need to find the newest, valid TMM NFO MediaFile nfo = new MediaFile(); for (MediaFile mf : movie.getMediaFiles(MediaFileType.NFO)) { if (mf.getFiledate() >= nfo.getFiledate() && MovieConnectors.isValidNFO(mf.getFileAsPath())) { nfo = new MediaFile(mf); } } if (nfo.getFiledate() > 0) { // one valid found? copy our NFO to all variants ArrayList<MediaFile> newNFOs = generateFilename(movie, nfo, newVideoBasename); // 1:N if (newNFOs.size() > 0) { // ok, at least one has been set up for (MediaFile newNFO : newNFOs) { boolean ok = copyFile(nfo.getFileAsPath(), newNFO.getFileAsPath()); if (ok) { needed.add(newNFO); } } } else { // list was empty, so even remove this NFO cleanup.add(nfo); } } else { LOGGER.trace("No valid NFO found for this movie"); } // now iterate over all non-tmm NFOs, and add them for cleanup or not for (MediaFile mf : movie.getMediaFiles(MediaFileType.NFO)) { if (MovieConnectors.isValidNFO(mf.getFileAsPath())) { cleanup.add(mf); } else { if (MovieModuleManager.MOVIE_SETTINGS.isMovieRenamerNfoCleanup()) { cleanup.add(mf); } else { needed.add(mf); } } } // ###################################################################### // ## rename all other types (copy 1:1) // ###################################################################### mfs = new ArrayList<>(); mfs.addAll(movie.getMediaFilesExceptType(MediaFileType.VIDEO, MediaFileType.NFO, MediaFileType.POSTER, MediaFileType.FANART, MediaFileType.SUBTITLE)); mfs.removeAll(Collections.singleton(null)); // remove all NULL ones! for (MediaFile other : mfs) { LOGGER.trace("Rename 1:1 " + other.getType() + " " + other.getFileAsPath()); ArrayList<MediaFile> newMFs = generateFilename(movie, other, newVideoBasename); // 1:N newMFs.removeAll(Collections.singleton(null)); // remove all NULL ones! for (MediaFile newMF : newMFs) { boolean ok = copyFile(other.getFileAsPath(), newMF.getFileAsPath()); if (ok) { needed.add(newMF); } else { // FIXME: what to do? not copied/exception... keep it for now... needed.add(other); } } } // ###################################################################### // ## rename subtitles later, but ADD it to not clean up // ###################################################################### needed.addAll(movie.getMediaFiles(MediaFileType.SUBTITLE)); // ###################################################################### // ## invalidade image cache // ###################################################################### for (MediaFile gfx : movie.getMediaFiles()) { if (gfx.isGraphic()) { ImageCache.invalidateCachedImage(gfx.getFileAsPath()); } } // remove duplicate MediaFiles Set<MediaFile> newMFs = new LinkedHashSet<>(needed); needed.clear(); needed.addAll(newMFs); movie.removeAllMediaFiles(); movie.addToMediaFiles(needed); movie.setPath(newPathname); // update .actors for (MovieActor actor : movie.getActors()) { actor.setEntityRoot(newPathname); } movie.saveToDb(); // cleanup & rename subtitle files renameSubtitles(movie); movie.gatherMediaFileInformation(false); // rewrite NFO if it's a MP NFO and there was a change with poster/fanart if (MovieModuleManager.MOVIE_SETTINGS.getMovieConnector() == MovieConnectors.MP && (posterRenamed || fanartRenamed)) { movie.writeNFO(); } movie.saveToDb(); // ###################################################################### // ## CLEANUP - delete all files marked for cleanup, which are not "needed" // ###################################################################### LOGGER.info("Cleanup..."); for (int i = cleanup.size() - 1; i >= 0; i--) { // cleanup files which are not needed if (!needed.contains(cleanup.get(i))) { MediaFile cl = cleanup.get(i); if (cl.getFileAsPath().equals(Paths.get(movie.getDataSource())) || cl.getFileAsPath().equals(movie.getPathNIO()) || cl.getFileAsPath().equals(Paths.get(oldPathname))) { LOGGER.warn("Wohoo! We tried to remove complete datasource / movie folder. Nooo way...! " + cl.getType() + ": " + cl.getFileAsPath()); // happens when iterating eg over the getNFONaming and we return a "" string. // then the path+filename = movie path and we want to delete :/ // do not show an error anylonger, just silently ignore... // MessageManager.instance.pushMessage(new Message(MessageLevel.ERROR, cl.getFile(), "message.renamer.failedrename")); // return; // rename failed continue; } if (Files.exists(cl.getFileAsPath())) { // unneeded, but for not displaying wrong deletes in logger... LOGGER.debug("Deleting " + cl.getFileAsPath()); Utils.deleteFileWithBackup(cl.getFileAsPath(), movie.getDataSource()); } try (DirectoryStream<Path> directoryStream = Files .newDirectoryStream(cl.getFileAsPath().getParent())) { if (!directoryStream.iterator().hasNext()) { // no iterator = empty LOGGER.debug("Deleting empty Directory " + cl.getFileAsPath().getParent()); Files.delete(cl.getFileAsPath().getParent()); // do not use recursive her } } catch (IOException ex) { } } } if (downloadMissingArtworks) { LOGGER.debug("Yay - movie upgrade :) download missing artworks"); MovieArtworkHelper.downloadMissingArtwork(movie); } }
From source file:org.fsl.roms.service.action.RoadCheckServiceAction.java
public void selectWarningNotice(RequestContext context) { ArrayList<Integer> selectedOffOuutes = (ArrayList<Integer>) context.getFlowScope() .get("offenceOutcomesSelected"); ArrayList<OffenceOutcomeView> offOuutes = (ArrayList<OffenceOutcomeView>) context.getFlowScope() .get("offenceOutcomes"); logger.info("selectedOffOuutes: " + selectedOffOuutes.size()); if (offOuutes != null && selectedOffOuutes != null) { for (OffenceOutcomeView offOut : offOuutes) { if (selectedOffOuutes.contains(offOut.getOffenceId())) { logger.info("offOut.getOffenceId(): " + offOut.getOffenceId()); logger.info("offOut.isIssuewWarningNotice(): " + offOut.isIssuewWarningNotice()); if (offOut.isIssuewWarningNotice()) { offOut.setIssuewWarningNotice(true); offOut.setIssueSummons(true); }/*from w w w .j a va 2s . c o m*/ } } } context.getFlowScope().put("offenceOutcomes", offOuutes); selectedOffOuutes.removeAll(selectedOffOuutes);// to be removed added because row closes context.getFlowScope().put("offenceOutcomesSelected", selectedOffOuutes);// to be removed }