Java tutorial
/* * * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. * */ package org.apache.flex.compiler.internal.parsing.as; import static org.apache.flex.compiler.common.ISourceLocation.UNKNOWN; import static org.apache.flex.compiler.internal.parsing.as.ASTokenTypes.*; import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; import java.io.Reader; import java.io.StringReader; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.EnumSet; import java.util.LinkedHashSet; import java.util.List; import java.util.Set; import java.util.Stack; import org.apache.flex.abc.semantics.ECMASupport; import org.apache.flex.compiler.parsing.GenericTokenStream; import org.apache.flex.compiler.parsing.IASToken; import org.apache.flex.compiler.parsing.IASToken.ASTokenKind; import org.apache.flex.compiler.problems.InvalidConfigLocationProblem; import org.apache.flex.compiler.problems.NonConstConfigVarProblem; import org.apache.flex.compiler.problems.ShadowedConfigNamespaceProblem; import org.apache.commons.io.IOUtils; import antlr.ANTLRException; import antlr.LLkParser; import antlr.MismatchedTokenException; import antlr.NoViableAltException; import antlr.ParserSharedInputState; import antlr.RecognitionException; import antlr.Token; import antlr.TokenBuffer; import antlr.TokenStream; import antlr.TokenStreamException; import org.apache.flex.compiler.asdoc.IASDocComment; import org.apache.flex.compiler.asdoc.IASParserASDocDelegate; import org.apache.flex.compiler.common.IFileSpecificationGetter; import org.apache.flex.compiler.common.ISourceLocation; import org.apache.flex.compiler.common.SourceLocation; import org.apache.flex.compiler.constants.IASKeywordConstants; import org.apache.flex.compiler.constants.IASLanguageConstants; import org.apache.flex.compiler.constants.IMetaAttributeConstants; import org.apache.flex.compiler.filespecs.IFileSpecification; import org.apache.flex.compiler.internal.filespecs.StringFileSpecification; import org.apache.flex.compiler.internal.parsing.TokenBase; import org.apache.flex.compiler.internal.scopes.ASScope; import org.apache.flex.compiler.internal.semantics.PostProcessStep; import org.apache.flex.compiler.internal.tree.as.ArrayLiteralNode; import org.apache.flex.compiler.internal.tree.as.BaseDefinitionNode; import org.apache.flex.compiler.internal.tree.as.BinaryOperatorNodeBase; import org.apache.flex.compiler.internal.tree.as.BlockNode; import org.apache.flex.compiler.internal.tree.as.ClassNode; import org.apache.flex.compiler.internal.tree.as.ConfigConstNode; import org.apache.flex.compiler.internal.tree.as.ConfigExpressionNode; import org.apache.flex.compiler.internal.tree.as.ContainerNode; import org.apache.flex.compiler.internal.tree.as.EmbedNode; import org.apache.flex.compiler.internal.tree.as.ExpressionNodeBase; import org.apache.flex.compiler.internal.tree.as.FileNode; import org.apache.flex.compiler.internal.tree.as.FullNameNode; import org.apache.flex.compiler.internal.tree.as.FunctionNode; import org.apache.flex.compiler.internal.tree.as.FunctionObjectNode; import org.apache.flex.compiler.internal.tree.as.IdentifierNode; import org.apache.flex.compiler.internal.tree.as.ImportNode; import org.apache.flex.compiler.internal.tree.as.LiteralNode; import org.apache.flex.compiler.internal.tree.as.MemberAccessExpressionNode; import org.apache.flex.compiler.internal.tree.as.ModifierNode; import org.apache.flex.compiler.internal.tree.as.NamespaceAccessExpressionNode; import org.apache.flex.compiler.internal.tree.as.NamespaceIdentifierNode; import org.apache.flex.compiler.internal.tree.as.NamespaceNode; import org.apache.flex.compiler.internal.tree.as.NodeBase; import org.apache.flex.compiler.internal.tree.as.QualifiedNamespaceExpressionNode; import org.apache.flex.compiler.internal.tree.as.ScopedBlockNode; import org.apache.flex.compiler.internal.tree.as.UnaryOperatorNodeBase; import org.apache.flex.compiler.internal.tree.as.VariableNode; import org.apache.flex.compiler.internal.tree.as.metadata.MetaTagsNode; import org.apache.flex.compiler.internal.workspaces.Workspace; import org.apache.flex.compiler.mxml.IMXMLTextData; import org.apache.flex.compiler.problems.AttributesNotAllowedOnPackageDefinitionProblem; import org.apache.flex.compiler.problems.CanNotInsertSemicolonProblem; import org.apache.flex.compiler.problems.EmbedInitialValueProblem; import org.apache.flex.compiler.problems.EmbedMultipleMetaTagsProblem; import org.apache.flex.compiler.problems.EmbedOnlyOnClassesAndVarsProblem; import org.apache.flex.compiler.problems.EmbedUnsupportedTypeProblem; import org.apache.flex.compiler.problems.ExpectDefinitionKeywordAfterAttributeProblem; import org.apache.flex.compiler.problems.ExtraCharactersAfterEndOfProgramProblem; import org.apache.flex.compiler.problems.FileNotFoundProblem; import org.apache.flex.compiler.problems.ICompilerProblem; import org.apache.flex.compiler.problems.InternalCompilerProblem; import org.apache.flex.compiler.problems.InternalCompilerProblem2; import org.apache.flex.compiler.problems.InvalidAttributeProblem; import org.apache.flex.compiler.problems.InvalidLabelProblem; import org.apache.flex.compiler.problems.InvalidTypeProblem; import org.apache.flex.compiler.problems.MXMLInvalidDatabindingExpressionProblem; import org.apache.flex.compiler.problems.MissingLeftBraceBeforeFunctionBodyProblem; import org.apache.flex.compiler.problems.MultipleConfigNamespaceDecorationsProblem; import org.apache.flex.compiler.problems.MultipleNamespaceAttributesProblem; import org.apache.flex.compiler.problems.MultipleReservedNamespaceAttributesProblem; import org.apache.flex.compiler.problems.NamespaceAttributeNotAllowedProblem; import org.apache.flex.compiler.problems.NestedClassProblem; import org.apache.flex.compiler.problems.NestedInterfaceProblem; import org.apache.flex.compiler.problems.NestedPackageProblem; import org.apache.flex.compiler.problems.ParserProblem; import org.apache.flex.compiler.problems.SyntaxProblem; import org.apache.flex.compiler.problems.UnboundMetadataProblem; import org.apache.flex.compiler.problems.UnexpectedEOFProblem; import org.apache.flex.compiler.problems.UnexpectedTokenProblem; import org.apache.flex.compiler.problems.XMLOpenCloseTagNotMatchProblem; import org.apache.flex.compiler.projects.IASProject; import org.apache.flex.compiler.tree.ASTNodeID; import org.apache.flex.compiler.tree.as.IASNode; import org.apache.flex.compiler.tree.as.IExpressionNode; import org.apache.flex.compiler.tree.as.IFileNodeAccumulator; import org.apache.flex.compiler.tree.as.IIdentifierNode; import org.apache.flex.compiler.tree.as.ILiteralNode.LiteralType; import org.apache.flex.compiler.tree.as.INamespaceDecorationNode; import org.apache.flex.compiler.tree.as.INamespaceDecorationNode.NamespaceDecorationKind; import org.apache.flex.compiler.tree.as.IOperatorNode.OperatorType; import org.apache.flex.compiler.tree.metadata.IMetaTagNode; import org.apache.flex.compiler.tree.metadata.IMetaTagsNode; import org.apache.flex.compiler.units.ICompilationUnit; import org.apache.flex.compiler.units.IInvisibleCompilationUnit; import org.apache.flex.compiler.workspaces.IWorkspace; import org.apache.flex.utils.FilenameNormalization; import org.apache.flex.utils.NonLockingStringReader; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; /** * Base class for the ANTLR-generated ActionScript parser {@link ASParser}. * Complex Java action code should be put in this base class instead of in the * ANTLR grammar. */ abstract class BaseASParser extends LLkParser implements IProblemReporter { private static final String CONFIG_AS = "config.as"; /** * Variable to use as an empty project configuration */ public static final IProjectConfigVariables EMPTY_CONFIGURATION = null; /** * Used to specify "directive" rule's end token as "no end token" - the * parser will consume till the end of the input stream when recover from * errors. */ protected static final int NO_END_TOKEN = -1; private static final String SUB_SYSTEM = "ASParser"; /** * Produces an AST from the given file input. A FileNode will always be * returned, however it is not guaranteed to contain any content. This is * provided as a convenience, as it is equivalent to calling: * <code>parseFile(spec, workspace, EnumSet.of(PostProcessStep.CALCULATE_OFFSETS), true, true)</code> * meaning we follow includes, fix the tree and calculate offsets. * * @param spec the {@link IFileSpecification} that points to the file that * will be parsed * @param fileSpecGetter the {@link IFileSpecificationGetter} that should be * used to open files. * @return a {@link FileNode} built from the given input */ public static FileNode parseFile(IFileSpecification spec, IFileSpecificationGetter fileSpecGetter) { return parseFile(spec, fileSpecGetter, EnumSet.of(PostProcessStep.CALCULATE_OFFSETS), EMPTY_CONFIGURATION, true); } /** * Produces an AST from the given file input. A FileNode will always be * returned, however it is not guaranteed to contain any content * * @param spec the {@link IFileSpecification} that points to the file that * will be parsed * @param fileSpecGetter the {@link IFileSpecificationGetter} that should be * used to open files. * @param postProcess the set of operations we want to perform on this tree * before it is returned. See {@link PostProcessStep} * @param variables the {@link IProjectConfigVariables} containing * project-level conditional compilation variables * @param followIncludes flag to determine if include statements should be * followed * @return a {@link FileNode} built from the given input */ public static FileNode parseFile(IFileSpecification spec, IFileSpecificationGetter fileSpecGetter, EnumSet<PostProcessStep> postProcess, IProjectConfigVariables variables, boolean followIncludes) { return parseFile(spec, fileSpecGetter, postProcess, variables, followIncludes, true, Collections.<String>emptyList(), DeferFunctionBody.DISABLED, null, null); } /** * Produces an AST from the given file input. A FileNode will always be * returned, however it is not guaranteed to contain any content * * @param spec the {@link IFileSpecification} that points to the file that * will be parsed * @param postProcess the set of operations we want to perform on this tree * before it is returned. See {@link PostProcessStep} * @param variables the {@link IProjectConfigVariables} containing * project-level conditional compilation variables * @param followIncludes flag to determine if include statements should be * followed * @param allowEmbeds flag to indicate if we should ignore embed meta data * or create EmbedNodes * @param includedFiles Files included by {@code asc -in} option. * @param flashProject Used to resolve included files in all the source * folders. Use {@code null} if the project is not a {@link IASProject}. * @param compilationUnit used to manage missing include files. both project * and compilation unit must be passed if you wish to have compilation units * re-built when their missing referenced files are added to the project. * @return a {@link FileNode} built from the given input */ public static FileNode parseFile(IFileSpecification spec, IFileSpecificationGetter fileSpecGetter, EnumSet<PostProcessStep> postProcess, IProjectConfigVariables variables, boolean followIncludes, boolean allowEmbeds, List<String> includedFiles, DeferFunctionBody deferFunctionBody, IASProject flashProject, ICompilationUnit compilationUnit) { assert spec != null : "File spec can't be null."; assert fileSpecGetter != null : "File specification getter can't be null"; assert fileSpecGetter.getWorkspace() instanceof Workspace : "Expected Workspace type."; final FileNode node = new FileNode(fileSpecGetter, spec.getPath()); final IncludeHandler includeHandler = node.getIncludeHandler(); includeHandler.setProjectAndCompilationUnit(flashProject, compilationUnit); StreamingASTokenizer tokenizer = null; try { tokenizer = StreamingASTokenizer.createForASParser(spec, includeHandler, followIncludes, includedFiles); final IRepairingTokenBuffer buffer = new StreamingTokenBuffer(tokenizer); final ASParser parser = new ASParser(fileSpecGetter.getWorkspace(), buffer); parser.deferFunctionBody = deferFunctionBody; parser.setProjectConfigVariables(variables); parser.setFilename(spec.getPath()); parser.setAllowEmbeds(allowEmbeds); parser.parseFile(node, postProcess); if (node.getAbsoluteEnd() < tokenizer.getEndOffset()) node.setEnd(tokenizer.getEndOffset()); node.setProblems(tokenizer.getTokenizationProblems()); node.setProblems(parser.getSyntaxProblems()); final OffsetLookup offsetLookup = new OffsetLookup(includeHandler.getOffsetCueList()); node.setOffsetLookup(offsetLookup); int[] absoluteOffset = offsetLookup.getAbsoluteOffset(spec.getPath(), tokenizer.getEndOffset()); //Absolute offset for the last token in a file shouldn't be more than one. assert absoluteOffset.length == 1 : "There seems to be a cycle in the include tree which has not been handled."; if (node.getAbsoluteEnd() < absoluteOffset[0]) node.setEnd(absoluteOffset[0]); } catch (FileNotFoundException e) { // Use the message from the FileNotFoundException exception as that // will have the actual file that is missing in cases where .as files // are combined. node.addProblem(new FileNotFoundProblem(e.getMessage())); } catch (Throwable e2) { ICompilerProblem problem = new InternalCompilerProblem2(spec.getPath(), e2, SUB_SYSTEM); node.addProblem(problem); } finally { IOUtils.closeQuietly(tokenizer); } return node; } /** * Parse a fragment of ActionScript. The resulting AST node is the return * value. The resulting definitions and scopes will be attached to the given * {@code containingScope}. This is used by MXML script tags. * * @param fragment ActionScript block. * @param path Containing source path of the ActionScript fragment. * @param offset Start offset of the script. * @param line Line offset of the script. * @param column Column offset of the script. * @param problems Problems parsing the script. * @param workspace Owner workspace. * @param fileNodeAccumulator Collect data that needs to be stored on * {@code FileNode}. * @param containingScope The resulting definitions and scopes from the * script will be attached to this scope. * @param variables Project config variables. * @param postProcess Post process steps. * @param followIncludes True if includes are followed. * @param includeHandler The include handler to use. * @return The resulting AST subtree generated from the ActionScript * fragment. */ public static ScopedBlockNode parseFragment2(final String fragment, final String path, final int offset, final int line, final int column, final Collection<ICompilerProblem> problems, final IWorkspace workspace, final IFileNodeAccumulator fileNodeAccumulator, final ASScope containingScope, final IProjectConfigVariables variables, final EnumSet<PostProcessStep> postProcess, final boolean followIncludes, final IncludeHandler includeHandler) { assert fragment != null; assert path != null; assert workspace != null; assert includeHandler != null; assert problems != null; StreamingASTokenizer tokenizer = null; ASParser parser = null; final IFileSpecification textFileSpec = new StringFileSpecification(path, fragment); final ScopedBlockNode container = new ScopedBlockNode(); container.setScope(containingScope); try { tokenizer = StreamingASTokenizer.create(textFileSpec, includeHandler); tokenizer.setSourcePositionAdjustment(offset, line, column); final IRepairingTokenBuffer buffer = new StreamingTokenBuffer(tokenizer); parser = new ASParser(workspace, buffer); parser.setFileNodeAccumulator(fileNodeAccumulator); parser.setFilename(path); parser.setProjectConfigVariables(variables); // Initialize depth of {...} to be positive number so that nested // package/class definitions can be detected. ((BaseASParser) parser).blockDepth = 1; while (buffer.LA(1) != ASTokenTypes.EOF) parser.directive(container, NO_END_TOKEN); problems.addAll(tokenizer.getTokenizationProblems()); problems.addAll(parser.getSyntaxProblems()); problems.addAll(container.runPostProcess(postProcess, containingScope)); } catch (RecognitionException e) { parser.consumeParsingError(e); problems.addAll(parser.getSyntaxProblems()); } catch (TokenStreamException e) { ICompilerProblem problem = genericParserProblem(path, offset, offset, line, column); problems.add(problem); } catch (FileNotFoundException e) { ICompilerProblem problem = genericParserProblem(path, offset, offset, line, column); problems.add(problem); } finally { if (parser != null) parser.disconnect(); IOUtils.closeQuietly(tokenizer); if (includeHandler != null && tokenizer != null) includeHandler.leaveFile(tokenizer.getEndOffset()); } return container; } /** * Parser entry-point for rebuild function body nodes. * * @param container Function body container node. * @param reader Function body reader. * @param path Source path. * @param blockOpenToken "{" of the function body. * @param problems Compiler problems. * @param workspace Current workspace. * @param fileNode AS file node. * @param configProcessor Configuration variables. */ public static void parseFunctionBody(final ScopedBlockNode container, final Reader reader, final String path, final ASToken blockOpenToken, final Collection<ICompilerProblem> problems, final IWorkspace workspace, final FileNode fileNode, final ConfigProcessor configProcessor) { assert container != null; assert reader != null; assert path != null; assert blockOpenToken != null; assert problems != null; assert workspace != null; assert fileNode != null; assert configProcessor != null; StreamingASTokenizer tokenizer = null; ASParser parser = null; try { tokenizer = new StreamingASTokenizer(); tokenizer.setReader(reader); tokenizer.setPath(path); tokenizer.setSourcePositionAdjustment(blockOpenToken.getEnd(), blockOpenToken.getLine(), blockOpenToken.getColumn()); final IRepairingTokenBuffer buffer = new StreamingTokenBuffer(tokenizer); parser = new ASParser(workspace, buffer); parser.setFileNodeAccumulator(fileNode); parser.setFilename(path); parser.setConfigProcessor(configProcessor); // Initialize "{..}" depth to positive number so that nested "package" // problems can be detected. ((BaseASParser) parser).blockDepth = 1; while (buffer.LA(1) != ASTokenTypes.EOF && buffer.LA(1) != ASTokenTypes.TOKEN_BLOCK_CLOSE) parser.directive(container, ASTokenTypes.TOKEN_BLOCK_CLOSE); problems.addAll(tokenizer.getTokenizationProblems()); problems.addAll(parser.getSyntaxProblems()); } catch (RecognitionException e) { parser.consumeParsingError(e); problems.addAll(parser.getSyntaxProblems()); } catch (TokenStreamException e) { problems.add(new ParserProblem(container)); } finally { if (parser != null) parser.disconnect(); IOUtils.closeQuietly(tokenizer); } } private static ICompilerProblem genericParserProblem(String path, int start, int end, int line, int column) { ISourceLocation location = new SourceLocation(path, start, end, line, column); return new ParserProblem(location); } /** * Parse a script tag with inline ActionScript. This function does not * trigger "enter" and "leave" event for the inlined script, because the * script's token offset is still in the same offset space with its parent * document. * <p> * Both {@code MXMLScopeBuilder} and {@code MXMLScriptNode} use * this parser entry point. Different post-process tasks are requested. */ public static ScopedBlockNode parseInlineScript(final IFileNodeAccumulator fileNodeAccumulator, final IMXMLTextData mxmlTextData, final Collection<ICompilerProblem> problems, final ASScope containingScope, final IProjectConfigVariables variables, final IncludeHandler includeHandler, final EnumSet<PostProcessStep> postProcess) { assert mxmlTextData != null : "MXMLTextData can't be null."; assert includeHandler != null : "IncludeHandler can't be null."; assert problems != null : "Problem container can't be null"; // create a file specification for the script from MXML tag data final String scriptSourcePath = mxmlTextData.getParent().getPath(); final String scriptContent = mxmlTextData.getCompilableText(); final Reader scriptReader = new StringReader(scriptContent); // source adjustment final int compilableTextStart = mxmlTextData.getCompilableTextStart(); final int compilableTextLine = mxmlTextData.getCompilableTextLine(); final int compilableTextColumn = mxmlTextData.getCompilableTextColumn(); final ScopedBlockNode container = new ScopedBlockNode(); // create lexer final StreamingASTokenizer tokenizer = StreamingASTokenizer.createForInlineScriptScopeBuilding(scriptReader, scriptSourcePath, includeHandler, compilableTextStart, compilableTextLine, compilableTextColumn); final IRepairingTokenBuffer buffer = new StreamingTokenBuffer(tokenizer); // create parser final ASParser parser = new ASParser(containingScope.getWorkspace(), buffer); try { // parse script parser.setFilename(scriptSourcePath); parser.setProjectConfigVariables(variables); parser.setFileNodeAccumulator(fileNodeAccumulator); while (buffer.LA(1) != ASTokenTypes.EOF) parser.directive(container, NO_END_TOKEN); problems.addAll(tokenizer.getTokenizationProblems()); problems.addAll(parser.getSyntaxProblems()); // attach to given outer scope container.setScope(containingScope); // run post-processes final Collection<ICompilerProblem> postProcessProblems = container.runPostProcess(postProcess, containingScope); problems.addAll(postProcessProblems); } catch (RecognitionException e) { parser.consumeParsingError(e); problems.addAll(parser.getSyntaxProblems()); } catch (TokenStreamException e) { ICompilerProblem problem = genericParserProblem(scriptSourcePath, mxmlTextData.getAbsoluteStart(), mxmlTextData.getAbsoluteEnd(), mxmlTextData.getLine(), mxmlTextData.getColumn()); problems.add(problem); } finally { parser.disconnect(); IOUtils.closeQuietly(tokenizer); } return container; } /** * Parse an expression from the passed in Reader. This method can handle * expressions that are inline (such as the body of a Function tag) or * expressions that just come from a java String. For the inline case, the * Reader should be a SourceFragmentsReader - it is up to the caller to set * up the SourceFragmentsReader correctly - see * MXMLExpressionNodeBase.parseExpressionFromTag. For the String case, the * Reader can just be a StringReader. This will parse an Expression from the * Reader and return the ExpressionNodeBase. It will return null, if there * is no expression, or it can't be parsed. */ public static ExpressionNodeBase parseExpression(final IWorkspace workspace, final Reader scriptReader, final Collection<ICompilerProblem> problems, final IProjectConfigVariables variables, final ISourceLocation location) { assert scriptReader != null : "reader can't be null."; assert problems != null : "Problem container can't be null"; String sourcePath = location.getSourcePath(); final ParserProblem genericParserProblem = new ParserProblem(location); // create tokenizer final StreamingASTokenizer tokenizer = StreamingASTokenizer.createForInlineExpressionParsing(scriptReader, sourcePath); final IRepairingTokenBuffer buffer = new StreamingTokenBuffer(tokenizer); // create parser final ASParser parser = new ASParser(workspace, buffer); ExpressionNodeBase expressionNode = null; try { // parse script parser.setFilename(sourcePath); parser.setProjectConfigVariables(variables); if (buffer.LA(1) != ASTokenTypes.EOF) expressionNode = parser.expression(); problems.addAll(tokenizer.getTokenizationProblems()); problems.addAll(parser.getSyntaxProblems()); } catch (RecognitionException e) { parser.consumeParsingError(e); problems.addAll(parser.getSyntaxProblems()); } catch (TokenStreamException e) { problems.add(genericParserProblem); } finally { parser.disconnect(); IOUtils.closeQuietly(tokenizer); } return expressionNode; } /** * Parses a string into a name valid in ActionScript. If the code is not * valid or does not yield an identifier, null will be returned * * @param fragment the string that contains a name * @return an {@link IIdentifierNode}, or null */ public static final IASNode[] parseProjectConfigVariables(IWorkspace workspace, String fragment, Collection<ICompilerProblem> problems) { ScopedBlockNode container = new ScopedBlockNode(); ASParser parser = null; try { IFileSpecification fileSpec = new StringFileSpecification(CONFIG_AS, fragment, 0); IncludeHandler includeHandler = new IncludeHandler(workspace); StreamingASTokenizer tokenizer = StreamingASTokenizer.create(fileSpec, includeHandler); tokenizer.setReader(new NonLockingStringReader(fragment)); tokenizer.setPath(CONFIG_AS); tokenizer.setFollowIncludes(false); final IRepairingTokenBuffer buffer = new StreamingTokenBuffer(tokenizer); parser = new ASParser(workspace, buffer, true); parser.setFilename(CONFIG_AS); while (buffer.LA(1) != ASTokenTypes.EOF) parser.directive(container, NO_END_TOKEN); problems.addAll(tokenizer.getTokenizationProblems()); problems.addAll(parser.getSyntaxProblems()); return parser.getConfigProcessorResults(); } catch (FileNotFoundException e) { assert false : "StringFileSpecification never raises this exception"; } catch (ANTLRException e) { // Ignore any parsing errors. } catch (RuntimeException e) { String path = parser.getSourceFilePath(); ICompilerProblem problem = (path == null) ? new InternalCompilerProblem(e) : new InternalCompilerProblem2(path, e, SUB_SYSTEM); parser.errors.add(problem); } finally { if (parser != null) parser.disconnect(); } int n = container.getChildCount(); IASNode[] children = new IASNode[n]; for (int i = 0; i < n; i++) { children[i] = container.getChild(i); } return children; } /** * Parses a databinding expression. */ public static final IExpressionNode parseDataBinding(IWorkspace workspace, Reader reader, Collection<ICompilerProblem> problems) { StreamingASTokenizer tokenizer = new StreamingASTokenizer(); tokenizer.setReader(reader); IRepairingTokenBuffer buffer = new StreamingTokenBuffer(tokenizer); ASParser parser = new ASParser(workspace, buffer); FileNode fileNode = new FileNode(workspace); // Parse the databinding and build children inside the FileNode. parser.parseFile(fileNode, EnumSet.noneOf(PostProcessStep.class)); // Run post-processing to calculate all offsets. // Without this, identifiers have the right offsets but operators don't. EnumSet<PostProcessStep> postProcessSteps = EnumSet.of(PostProcessStep.CALCULATE_OFFSETS); Collection<ICompilerProblem> postProcessProblems = fileNode.runPostProcess(postProcessSteps, null); problems.addAll(tokenizer.getTokenizationProblems()); problems.addAll(parser.getSyntaxProblems()); problems.addAll(postProcessProblems); int n = fileNode.getChildCount(); // If we didn't get any children of the file node, // we must have parsed whitespace (or nothing). // An databinding like {} or { } represents the empty string. if (n == 0) { return new LiteralNode(LiteralType.STRING, ""); } // If we got more than one child, report that the databinding expression // is invalid. It must be a single expression. else if (n > 1) { final ICompilerProblem problem = new MXMLInvalidDatabindingExpressionProblem(fileNode); problems.add(problem); return null; } IASNode firstChild = fileNode.getChild(0); // If we got a single child but it isn't an expression, // report a problem. if (!(firstChild instanceof IExpressionNode)) { final ICompilerProblem problem = new MXMLInvalidDatabindingExpressionProblem(fileNode); problems.add(problem); return null; } // We got a single expression, so return it. return (IExpressionNode) firstChild; } /** * Our version of a token buffer that allows us to handle optional * semicolons, etc */ protected IRepairingTokenBuffer buffer; /** * Current metadata attributes. Attributes go into this container, then are * pulled out when the corresponding class, function, or variable is parsed. */ protected MetaTagsNode currentAttributes; /** * {@link IASParserASDocDelegate} used to track ASDoc information. */ protected final IASParserASDocDelegate asDocDelegate; /** * Last token that produced an error */ protected Token errorToken = null; /** * Flag to determine if we should create EmbedNodes or ignore embed metadata */ protected boolean allowEmbeds = true; /** * flag that tracks whether the parser is currently inside of a class * definition this is "borrowed" from the old parser to deal with some ASDoc * functionality */ protected boolean insideClass = false; /** * Flag that indicates we are parsing a pseudo file that represents the * config vars on the command line (or from some other project configuration * option) */ private final boolean parsingProjectConfigVariables; /** * Cut down on object construction when we throw on the matches call */ private MismatchedTokenException exceptionPool = new MismatchedTokenException(tokenNames, null, null, false); /** * Errors we've collected as we're moving forward */ protected final Set<ICompilerProblem> errors = new LinkedHashSet<ICompilerProblem>(); /** * This object allows the parser to store data on a given {@code FileNode}. */ private IFileNodeAccumulator fileNodeAccumulator; /** * Config processor used to handle config namespace expressions */ private ConfigProcessor configProcessor; /** * Determines if we should allowe errors. Used in conditional compilation to * suppress blocks that are excluded */ private boolean allowErrorsInContext = true; /** * True if the current input source is an ActionScript file, as opposed to * AS3 scripts in MXML or other synthesized source fragments. */ protected DeferFunctionBody deferFunctionBody = DeferFunctionBody.DISABLED; /** * A secondary reader to capture function body text on-the-fly. This is an * optimization for large files in order to avoid {@link Reader#skip(long)} * when the function body is rebuilt from source file and start offset. * <p> * This optimization is turned off if there's an {@code include} directive * in the function body. */ private final Reader secondaryReader; /** * Character offset into the {@link #secondaryReader}. */ private int secondaryReaderPosition = 0; /** * Number of nested "packages". */ private int packageDepth = 0; /** * Number of nested "blocks". */ private int blockDepth = 0; /** * Number of nested "groups" in "group directives". */ private int groupDepth = 0; /** * A recursive descent parsing stack of XML tags. */ private final Stack<XMLTagInfo> xmlTags; /** * Create an ActionScript parser from a workspace and a token buffer. * * @param workspace Current workspace. * @param buffer Token buffer. */ protected BaseASParser(IWorkspace workspace, IRepairingTokenBuffer buffer) { this(workspace, buffer, false); } /** * This constrcutor should ONLY be used in the very special case of parsing * project config vars. Typically this is done as part of parsing the * command line. For normal case, use the two argument constructor * * @param workspace Current workspace. * @param buffer Token buffer. * @param parsingProjectConfigVariables true for special case when we are * parsing not text, but a fake "file" of configuration variables. */ protected BaseASParser(IWorkspace workspace, IRepairingTokenBuffer buffer, boolean parsingProjectConfigVariables) { super(new ParserSharedInputState(), 1); xmlTags = new Stack<XMLTagInfo>(); this.buffer = buffer; configProcessor = new ConfigProcessor(workspace, this); // If there are no include directives in the function body, we // can cache the function body text to save the seeking time. secondaryReader = tryGetSecondaryReader(workspace, buffer); this.asDocDelegate = workspace.getASDocDelegate().getASParserASDocDelegate(); this.parsingProjectConfigVariables = parsingProjectConfigVariables; } /** * Try to initialize {@link #secondaryReader}. If fails, this optimization * is not available for the current file. * * @param workspace Current workspace. * @param buffer Underlying token buffer. * @return Initialized secondary reader or null. */ private static Reader tryGetSecondaryReader(IWorkspace workspace, IRepairingTokenBuffer buffer) { if (buffer instanceof StreamingTokenBuffer) { final StreamingTokenBuffer streamingTokenBuffer = (StreamingTokenBuffer) buffer; // token without source path (probably from string literals if (streamingTokenBuffer.getSourcePath() == null) return null; // token source path doesn't exist: imaginary sources final String sourcePath = FilenameNormalization.normalize(streamingTokenBuffer.getSourcePath()); if (!new File(sourcePath).isFile()) return null; // try to create a reader from file specification final IFileSpecification fileSpec = workspace.getFileSpecification(sourcePath); if (fileSpec != null) { try { // success - optimization is available return fileSpec.createReader(); } catch (FileNotFoundException e) { return null; } } } return null; } protected BaseASParser(TokenBuffer tokenBuf, int k) { super(tokenBuf, k); throw new UnsupportedOperationException(); } protected BaseASParser(TokenStream lexer, int k) { super(lexer, k); throw new UnsupportedOperationException(); } protected BaseASParser(ParserSharedInputState state, int i) { super(state, i); throw new UnsupportedOperationException(); } protected final void addConditionalCompilationNamespace(final NamespaceNode node) { // Need to process the node even if there is an error so the node will // have the location information needed to display an error. configProcessor.addConditionalCompilationNamespace(node); if (!isGlobalContext()) { ICompilerProblem problem = new InvalidConfigLocationProblem(node); addProblem(problem); } } /** * Sets the {@link IProjectConfigVariables} to be used to support * conditional compilation * * @param variables {@link IProjectConfigVariables} for the given context */ public void setProjectConfigVariables(IProjectConfigVariables variables) { configProcessor.connect(variables); } /** * Bind the current parser to the given {@code ConfigProcessor}. * * @param configProcessor Resolve configuration variables at parse-time. */ protected final void setConfigProcessor(ConfigProcessor configProcessor) { assert configProcessor != null; configProcessor.setParser(this); this.configProcessor = configProcessor; } void addConfigConstNode(ConfigConstNode node) { configProcessor.addConfigConstNode(node); if (node.getKeywordNode().getKeywordId() != ASTokenTypes.TOKEN_KEYWORD_CONST) { ICompilerProblem problem = new NonConstConfigVarProblem(node.getNameExpressionNode()); addProblem(problem); } if (!isGlobalContext()) { ICompilerProblem problem = new InvalidConfigLocationProblem(node.getNameExpressionNode()); addProblem(problem); } } IASNode[] getConfigProcessorResults() { if (configProcessor != null) { return configProcessor.getConfigChildren(); } return new IASNode[0]; } protected final LiteralNode evaluateConstNodeExpression(final ConfigExpressionNode node) { return configProcessor.evaluateConstNodeExpression(node); } protected final boolean isConfigNamespace(final NamespaceIdentifierNode id) { return configProcessor.isConfigNamespace(id.getName()); } public Collection<ICompilerProblem> getSyntaxProblems() { return errors; } /** * Close the parser and release resources. */ protected final void disconnect() { if (configProcessor != null) configProcessor.detachParser(this); IOUtils.closeQuietly(secondaryReader); } void setFileNodeAccumulator(IFileNodeAccumulator fileNodeAccumulator) { this.fileNodeAccumulator = fileNodeAccumulator; } @Override public String getSourceFilePath() { return getFilename(); } protected void setAllowErrorsInContext(boolean allow) { allowErrorsInContext = allow; } @Override public void addProblem(ICompilerProblem problem) { if (allowErrorsInContext) errors.add(problem); } public void setAllowEmbeds(boolean allowEmbeds) { this.allowEmbeds = allowEmbeds; } /** * Assemble the tokens into a parse tree with a root FileNode. * * @param fileNode The parser builds AST into this root {@code FileNode}. * @param features Post-process steps to be performed. */ public void parseFile(FileNode fileNode, EnumSet<PostProcessStep> features) { currentAttributes = null; fileNode.setStart(0); fileNode.setLine(0); fileNode.setColumn(0); String sourcePath = fileNode.getSourcePath(); setFilename(sourcePath); exceptionPool.fileName = sourcePath; exceptionPool.mismatchType = MismatchedTokenException.RANGE; fileNodeAccumulator = fileNode; try { file(fileNode); } catch (RecognitionException e) { final ASToken current = buffer.LT(1); if (e instanceof NoViableAltException) { addProblem(new SyntaxProblem(current)); } else if (e instanceof MismatchedTokenException) { ASToken expected = new ASToken(((MismatchedTokenException) e).expecting, UNKNOWN, UNKNOWN, UNKNOWN, UNKNOWN, ""); addProblem(unexpectedTokenProblem(current, expected.getTokenKind())); } } catch (TokenStreamException e) { addProblem(new SyntaxProblem(fileNode, null)); } finally { disconnect(); } fileNode.processAST(features); } protected abstract void fileLevelDirectives(ContainerNode containerNode) throws RecognitionException, TokenStreamException; protected void encounteredImport(ImportNode importNode) { if (fileNodeAccumulator != null) { fileNodeAccumulator.addImportNode(importNode); } } /** * Parse a String of meta data from an @function, by changing a string such * as: <code>@Embed(source='a.png')</code> to: * <code>[Embed(source='a.png')]</code> and then running it through the meta * data parser * * @param metadataTagText The @function text. * @param problems The collection of compiler problems to which this method will add problems. * @return MetaTagsNode or null if error parsing the meta data */ public static MetaTagsNode parseAtFunction(IWorkspace workspace, String metadataTagText, String sourcePath, int start, int line, int column, Collection<ICompilerProblem> problems) { StringBuilder stringBuffer = new StringBuilder(); stringBuffer.append("["); stringBuffer.append(metadataTagText.substring(1)); stringBuffer.append("]"); return ASParser.parseMetadata(workspace, stringBuffer.toString(), sourcePath, start, line, column, problems); } /** * Parse metadata tags from a string. * * @param workspace Current workspace. * @param metadataContent Source text. * @param sourcePath Source path. * @param start Start offset. * @param line Line offset. * @param column Column offset. * @param problems Problem collection. * @return A {@code MetaTagsNode} containing all the parsed metadata tags. */ public static MetaTagsNode parseMetadata(final IWorkspace workspace, final String metadataContent, final String sourcePath, final int start, final int line, final int column, final Collection<ICompilerProblem> problems) { final long lastModified = workspace.getFileSpecification(sourcePath).getLastModified(); final IFileSpecification fileSpec = new StringFileSpecification(sourcePath, metadataContent, lastModified); final IncludeHandler includeHandler = new IncludeHandler(workspace); final ASParser parser = new ASParser(workspace, (IRepairingTokenBuffer) null); try { final StreamingASTokenizer tokenizer = StreamingASTokenizer.create(fileSpec, includeHandler); tokenizer.setSourcePositionAdjustment(start, line, column); for (ASToken asToken = tokenizer.next(); asToken != null; asToken = tokenizer.next()) { // Only metadata tokens and ASDoc tokens are relevant. switch (asToken.getType()) { case TOKEN_ATTRIBUTE: parser.parseMetadata(asToken, problems); break; case TOKEN_ASDOC_COMMENT: parser.asDocDelegate.setCurrentASDocToken(asToken); break; default: // Ignore other tokens. break; } } } catch (FileNotFoundException e) { // Do nothing. Source is from a string literal, so there should // never be FileNotFoundException. } return parser.currentAttributes; } /** * Parse metadata tags with a forked sub-parser. * * @param attributeToken Metadata token. * @param problems Problem collection. */ protected final void parseMetadata(Token attributeToken, Collection<ICompilerProblem> problems) { assert problems != null : "Expected problem collection."; assert attributeToken != null : "Expected attribute token."; // Extract metadata tokens. final List<MetadataToken> metadataTokens; if (attributeToken instanceof MetaDataPayloadToken) { metadataTokens = ((MetaDataPayloadToken) attributeToken).getPayload(); } else { final NonLockingStringReader metadataReader = new NonLockingStringReader(attributeToken.getText()); final MetadataTokenizer tokenizer = new MetadataTokenizer(metadataReader); tokenizer.setAdjust(((TokenBase) attributeToken).getStart()); metadataTokens = tokenizer.parseTokens(); } // Initialize metadata parser. final GenericTokenStream metadataTokenStream = new GenericTokenStream(metadataTokens); final MetadataParser metadataParser = new MetadataParser(metadataTokenStream); metadataParser.setASDocDelegate(asDocDelegate.getMetadataParserASDocDelegate()); // The parsed metadata tags will be added to this container node. if (currentAttributes == null) currentAttributes = new MetaTagsNode(); // Parse metadata. try { metadataParser.meta(currentAttributes); } catch (RecognitionException e) { final ParserProblem problem = new ParserProblem((TokenBase) attributeToken); problems.add(problem); } catch (TokenStreamException e) { //do nothing } } /** * Determines if the current metadata is found within a pure statement * context, or if it is going to be bound to a definition. If the item is * free-floating (not about to be bound to a definition) then we set its * parent as the passed in container, and log an error. * * @param attributeToken the token that contains the metadata * @param container the {@link ContainerNode} we will potentially add our * metadata to as a child */ protected final void preCheckMetadata(Token attributeToken, ContainerNode container) { if (currentAttributes == null) return; final int la = LA(1); if (ASToken.isDefinitionKeyword(la) || ASToken.isModifier(la) || isConfigCondition() || la == TOKEN_NAMESPACE_ANNOTATION || la == TOKEN_ASDOC_COMMENT || la == TOKEN_ATTRIBUTE) return; container.addItem(currentAttributes); final ICompilerProblem problem = new UnboundMetadataProblem((TokenBase) attributeToken); addProblem(problem); currentAttributes = null; } /** * Check if the look-ahead can be matched as a "ConfigCondition". * * @return True if the following input is "ConfigCondition". */ protected final boolean isConfigCondition() { return LA(1) == TOKEN_NAMESPACE_NAME && LA(2) == TOKEN_OPERATOR_NS_QUALIFIER && LA(3) == TOKEN_IDENTIFIER; } /** * Stores decorations on the given variable definition. This will set any * collected modifiers, namespace, metadata or comment we've encountered * * @param decoratedNode * @param node */ protected void storeVariableDecorations(VariableNode decoratedNode, ContainerNode node, INamespaceDecorationNode namespace, List<ModifierNode> modList) { storeDecorations(decoratedNode, node, namespace, modList); storeEmbedDecoration(decoratedNode, decoratedNode.getMetaTags()); } protected void storeEmbedDecoration(VariableNode variable, IMetaTagsNode metaTags) { if (!allowEmbeds) return; // no embed meta tag, so nothing more to do if (metaTags == null || !metaTags.hasTagByName(IMetaAttributeConstants.ATTRIBUTE_EMBED)) return; // can't have an initial value on an embed variable if (variable.getAssignedValueNode() != null) { addProblem(new EmbedInitialValueProblem(variable)); return; } // This type checking was ported from the old compiler in EmbedEvaluator.evaluate(Context, MetaDataNode) // Under normal circumstances type checking should be done on ITypeDefinition NOT string compares. // To make that happen, it should be done during codegen reduce_embed(). String typeName = variable.getTypeName(); if (!(IASLanguageConstants.Class.equals(typeName) || IASLanguageConstants.String.equals(typeName))) { addProblem(new EmbedUnsupportedTypeProblem(variable)); return; } IMetaTagNode[] embedMetaTags = metaTags.getTagsByName(IMetaAttributeConstants.ATTRIBUTE_EMBED); if (embedMetaTags.length > 1) { addProblem(new EmbedMultipleMetaTagsProblem(variable)); return; } EmbedNode embedNode = new EmbedNode(getFilename(), embedMetaTags[0], fileNodeAccumulator); variable.setAssignedValue(null, embedNode); } /** * Stores decorations on the given definition. This will set any collected * modifiers, namespace, metadata or comment we've encountered * * @param decoratedNode * @param node */ protected void storeDecorations(BaseDefinitionNode decoratedNode, ContainerNode node, INamespaceDecorationNode namespace, List<ModifierNode> modList) { //add the metadata if (currentAttributes != null) { IMetaTagNode[] embedTags = currentAttributes.getTagsByName(IMetaAttributeConstants.ATTRIBUTE_EMBED); assert embedTags != null; if (embedTags.length > 0) { // only member variables and classes can be annotated with embed data if (!((decoratedNode instanceof VariableNode) || (decoratedNode instanceof ClassNode))) { for (IMetaTagNode embedTag : embedTags) addProblem(new EmbedOnlyOnClassesAndVarsProblem(embedTag)); } } decoratedNode.setMetaTags(currentAttributes); } //add modifiers if (modList != null) { int size = modList.size(); for (int i = 0; i < size; i++) { ModifierNode modifierNode = modList.get(i); decoratedNode.addModifier(modifierNode); } } //set the namespace if (namespace != null) decoratedNode.setNamespace(namespace); final IASDocComment asDocComment = asDocDelegate.afterDefinition(decoratedNode); if (asDocComment != null) decoratedNode.setASDocComment(asDocComment); currentAttributes = null; } protected final boolean namespaceIsConfigNamespace(INamespaceDecorationNode node) { return node != null && node.getNamespaceDecorationKind() == NamespaceDecorationKind.CONFIG; } protected void logMultipleConfigNamespaceDecorationsError(NodeBase source) { ICompilerProblem problem = new MultipleConfigNamespaceDecorationsProblem(source); addProblem(problem); } /** * Check if the namespace conflicts with a config namespace * * @param ns the NamespaceNode to check */ protected void checkNamespaceDefinition(NamespaceNode ns) { if (configProcessor.isConfigNamespace(ns.getName())) addProblem(new ShadowedConfigNamespaceProblem(ns, ns.getName())); } /** * Create AST for various types of namespace access expressions. * * @param left left-hand side of {@code ::} is the namespace expression * @param op {@code ::} token * @param right right-hand side of {@code ::} is the variable * @return AST for the namespace access expressions. */ protected final ExpressionNodeBase transformToNSAccessExpression(ExpressionNodeBase left, ASToken op, ExpressionNodeBase right) { checkForChainedNamespaceQualifierProblem(op, right); final ExpressionNodeBase result; if (left instanceof FullNameNode) { // Left-hand side is a "full name", for example: "ns1::ns2::member". // Then convert the "full name" node into a namespace qualifier, // and associate it with the variable. final QualifiedNamespaceExpressionNode qualifier = new QualifiedNamespaceExpressionNode( (FullNameNode) left); result = new NamespaceAccessExpressionNode(qualifier, op, right); } else if (left instanceof MemberAccessExpressionNode) { // In this case, we need to turn the right side into the full qualified bit. IExpressionNode maRight = ((MemberAccessExpressionNode) left).getRightOperandNode(); if (maRight instanceof NamespaceIdentifierNode) { ((MemberAccessExpressionNode) left).setRightOperandNode( new NamespaceAccessExpressionNode((NamespaceIdentifierNode) maRight, op, right)); } else if (maRight instanceof IdentifierNode) { ((MemberAccessExpressionNode) left).setRightOperandNode(new NamespaceAccessExpressionNode( new NamespaceIdentifierNode((IdentifierNode) maRight), op, right)); //this is the @ case, so @x::y } else if (maRight instanceof UnaryOperatorNodeBase && ((UnaryOperatorNodeBase) maRight).getOperator() == OperatorType.AT) { ((UnaryOperatorNodeBase) maRight).setExpression(new NamespaceAccessExpressionNode( (IdentifierNode) ((UnaryOperatorNodeBase) maRight).getOperandNode(), op, right)); } if (maRight.hasParenthesis() && maRight instanceof MemberAccessExpressionNode) { ((MemberAccessExpressionNode) left).setRightOperandNode(new NamespaceAccessExpressionNode( new QualifiedNamespaceExpressionNode((MemberAccessExpressionNode) maRight), op, right)); } result = left; } else if (left instanceof NamespaceIdentifierNode && right instanceof IdentifierNode && ((NamespaceIdentifierNode) left) .getNamespaceDecorationKind() == NamespaceDecorationKind.CONFIG) { // Check to see if this is a "configCondition". final ConfigExpressionNode cn = new ConfigExpressionNode((NamespaceIdentifierNode) left, (ASToken) op, (IdentifierNode) right); result = evaluateConstNodeExpression(cn); } else { result = new NamespaceAccessExpressionNode(left, op, right); } return result; } /** * Check for syntax error of chained namespace qualifiers: * * <pre> * ns1::ns2::ns3::foo = 10; * </pre> */ protected final void checkForChainedNamespaceQualifierProblem(ASToken nsAccessOp, ExpressionNodeBase right) { if (nsAccessOp != null && right != null && right.getNodeID() == ASTNodeID.NamespaceIdentifierID) { final ICompilerProblem problem = new UnexpectedTokenProblem(nsAccessOp, ASTokenKind.IDENTIFIER); addProblem(problem); } } /** * Set the offsets of an empty BlockNode generated from a {} token. * * @param blockToken {} token * @param block BlockNode corresponding to that token */ protected void setOffsetsOfEmptyBlock(IASToken blockToken, BlockNode block) { if (blockToken.isImplicit()) { block.span((Token) blockToken); } else { block.span(blockToken.getStart() + 1, blockToken.getStart() + 1, blockToken.getLine(), blockToken.getColumn()); } } protected final void disableSemicolonInsertion() { buffer.setEnableSemicolonInsertion(false); } protected final void enableSemicolonInsertion() { buffer.setEnableSemicolonInsertion(true); } /** * Create a syntax error or warning based on the * {@code RecognitionException}. * * @param ex ANTLR-generated parsing exception. * @param endToken The expected end token for the currently parsing * fragment. It is used to determine the compiler problem type to create. */ protected final void consumeParsingError(RecognitionException ex, int endToken) { // Skip over inserted semicolon, because the fix-up token shouldn't // appear in the error message. final ASToken current = buffer.lookAheadSkipInsertedSemicolon(); final ICompilerProblem syntaxProblem; if (ex instanceof MismatchedTokenException) { final ASTokenKind expectedKind = ASToken.typeToKind(((MismatchedTokenException) ex).expecting); syntaxProblem = unexpectedTokenProblem(current, expectedKind); } else if (endToken != NO_END_TOKEN) { final ASTokenKind expectedKind = ASToken.typeToKind(endToken); syntaxProblem = unexpectedTokenProblem(current, expectedKind); } else { final ASToken errorToken = current.getType() == EOF ? buffer.previous() : current; syntaxProblem = new SyntaxProblem(errorToken); } addProblem(syntaxProblem); } /** * @see #consumeParsingError(RecognitionException, int) */ protected final void consumeParsingError(RecognitionException ex) { consumeParsingError(ex, NO_END_TOKEN); } /** * Force the parser to report a {@code MismatchedTokenException} when failed * to parse an identifier. The reason is that an identifier can either be an * {@code IDENTIFIER} token or one of the "contextual reserved words" like * {@code namespace}. Since we always want the error message to be * "Expected ... but got ....", we convert the exception here. * * @param ex {@code NoViableAltException} thrown in * {@link ASParser#identifier()}. * @return missing identifier */ protected IdentifierNode expectingIdentifier(NoViableAltException ex) { final MismatchedTokenException mismatchedTokenException = new MismatchedTokenException( new String[] { "IDENTIFIER" }, ex.token, ASTokenTypes.TOKEN_IDENTIFIER, false, ex.fileName); return handleMissingIdentifier(mismatchedTokenException); } /** * handles a case where we expected an identifier but our production * produced an error, while we were trying to produce a short name. If tree * fixing is enabled, this will return a new identifier to make the tree * "correct" * * @param ex the exception that was thrown * @return an {@link IdentifierNode} or null if tree fixing is not turned on */ protected IdentifierNode handleMissingIdentifier(RecognitionException ex) { consumeParsingError(ex); //we don't want to drop a semicolon here //now let's guard against the case where the very next token is an identifier. //if we produce an identifier here, everything downstream might fail final IdentifierNode node; ASToken current = buffer.previous(); ASToken la2 = LT(1 + 1); ASToken la1 = LT(1); if (current.getType() == ASTokenTypes.TOKEN_RESERVED_WORD_EXTENDS) { // Fix for CMP-1087: the following two branches are too greedy. // i.e. // public class T extends implements MyInterface // ^ // The "extends" keyword expects an identifier. However, instead of // reporting the missing identifier and continue with "implements", // the following recovery logic throws away "implements" keyword and // sends back "MyInterface". node = IdentifierNode.createEmptyIdentifierNodeAfterToken(current); } else if (la2.getType() == ASTokenTypes.TOKEN_IDENTIFIER && la2.getLine() == current.getLine()) { //let's make sure this is on the same line, avoiding possibly going past the end of the statement //since this is all repair code anyway, this produces a much "saner" repaired tree ASToken token = LT(1 + 1); node = new IdentifierNode(token.getText(), (IASToken) token); consume(); //consume error token consume(); //act as match() for identifier, which consumes it } else if (la1.isKeywordOrContextualReservedWord() && la1.getLine() == current.getLine()) { // If it's a keyword, repair by making an identifier node with the text of the keyword // This makes a more sensible tree - the user may be in the middle of typing an identifier that // starts with a keyword. They probably meant "f.is" rather than "(f.)is" for example node = new IdentifierNode(la1.getText(), (IASToken) la1); consume(); } else { node = IdentifierNode.createEmptyIdentifierNodeAfterToken(current); } return node; } /** * handles a case where we expected an identifier but our production * produced an error, while we were trying to produce a dotted name. If tree * fixing is enabled, this will return a new identifier to make the tree * "correct" * * @param ex the exception that was thrown * @return an {@link IdentifierNode} or null if tree fixing is not turned on */ protected ExpressionNodeBase handleMissingIdentifier(RecognitionException ex, ExpressionNodeBase n) { if (n instanceof FullNameNode) { ExpressionNodeBase temp = handleMissingIdentifier(ex); if (temp != null) { ((FullNameNode) n).setRightOperandNode(temp); } return n; } handleParsingError(ex); return n; } /** * Logs a syntax error against the given token * * @param badToken the token that caused the syntax error */ protected final void logSyntaxError(ASToken badToken) { addProblem(new SyntaxProblem(badToken)); } /** * Called by the parser to handle specific error cases we come across. If * the error handler can fix the error to potentially yield a successful * production then true is returned by this method. This method may insert * tokens into the token stream, or change the types of tokens that produced * the error. * * @param ex the exception thrown by the parser * @param endToken expected end token * @return true if this error case can potentially be fixed */ protected boolean handleParsingError(final RecognitionException ex, final int endToken) { final ASToken current = buffer.LT(1); // This variable will be assigned to "fErrorToken" after the error // recovery. We don't want inserted "virtual semicolons" to be come // an "error token", because it generates confusing syntax errors. final ASToken errorToken; if (current.getType() == TOKEN_SEMICOLON) errorToken = buffer.lookAheadSkipInsertedSemicolon(); else errorToken = current; final boolean result = handleParsingError(ex, current, errorToken, endToken); this.errorToken = errorToken; return result; } /** * @see #handleParsingError(RecognitionException, int) */ protected boolean handleParsingError(RecognitionException ex) { return handleParsingError(ex, NO_END_TOKEN); } private boolean handleParsingError(final RecognitionException ex, final ASToken current, final ASToken errorToken, final int endToken) { // Ignore ASDoc problems. if (current.getType() == ASTokenTypes.TOKEN_ASDOC_COMMENT) { consume(); return true; } // Don't log an error if we've seen this token already. if (errorToken != this.errorToken) consumeParsingError(ex, endToken); return recoverFromRecognitionException(ex, current); } /** * Generic routine to recover from an {@code RecognitionException}. * * @param ex ANTLR-generated parsing exception. * @param nextToken Next token. * @return True if error recovery succeeded. */ private boolean recoverFromRecognitionException(final RecognitionException ex, final ASToken nextToken) { // Don't recover from EOF, because an "unexpected EOF" problem should // have been logged already. if (nextToken.getType() == EOF) return false; // Same token is producing an error, consume it. if (nextToken == errorToken) { consume(); return true; } /* * if the token we had is a keyword, there is a chance that the user * could be in the middle of typing the name of an identifier that * starts with a keyword change it to an identifier, don't consume and * see if the parser can recover and turn it into a name the error, * however, will still be logged. */ if (nextToken.isKeywordOrContextualReservedWord()) { /* * There are a few cases where we don't want this to occur however. * Most notably, if the user is missing a close bracket or paren, * assume they are closing a structure and just return without * changing the type. Both of these can occur when adding metadata */ if (ex instanceof MismatchedTokenException) { switch (((MismatchedTokenException) ex).expecting) { case ASTokenTypes.TOKEN_SQUARE_CLOSE: case ASTokenTypes.TOKEN_PAREN_CLOSE: return true; } } nextToken.setType(ASTokenTypes.TOKEN_IDENTIFIER); return true; } if (nextToken.isOpenToken()) { // Don't have the inserted semicolon be the next token we are // looking for, since we're opening a statement. buffer.insertSemicolon(false); } else if (ASToken.isCloseToken(nextToken.getType())) { // Closed a block, semicolon will be matched later. buffer.insertSemicolon(true); } else if (nextToken.getType() == ASTokenTypes.TOKEN_SEMICOLON) { // Our semicolon isn't valid here, so don't bother inserting // another one. consume(); } else { buffer.insertSemicolon(true); } return true; } protected void endContainerAtError(RecognitionException ex, NodeBase node) { Token endToken = null; if (ex instanceof NoViableAltException) { endToken = ((NoViableAltException) ex).token; } else if (ex instanceof MismatchedTokenException) { endToken = ((MismatchedTokenException) ex).token; } if (endToken == null || endToken.getType() == ASTokenTypes.EOF) { endToken = buffer.previous(); } node.endBefore(endToken); } @Override public final void consume() { buffer.consume(); } /** * Match optional semicolon. * <p> * This method will report a {@link CanNotInsertSemicolonProblem} if a * "virtual semicolon" is expected but failed to be inserted, except when * there's already a syntax error on the same line, because the preceding * syntax error might early-terminate a statement, making the parser to * expect a optional semicolon. * <p> * It's not "wrong" to always report the semicolon problem. However, it * would make too much "noise" on the console, and it would break almost all * negative ASC tests. * * @return True if optional semicolon is matched. * @see IRepairingTokenBuffer#matchOptionalSemicolon() */ protected boolean matchOptionalSemicolon() { final boolean success = buffer.matchOptionalSemicolon(); if (!success) { final ICompilerProblem lastError = Iterables.getLast(errors, null); if (lastError == null || lastError.getLine() < buffer.previous().getLine()) { addProblem(new CanNotInsertSemicolonProblem(buffer.LT(1))); } } return success; } /** * An optional function body can either be a function block or a semicolon * (virtual semicolon). If neither is matched, a * {@link MissingLeftBraceBeforeFunctionBodyProblem} occurs. However, we * only report the problem if the function definition parsing doesn't have * other syntax issues so far. */ protected void reportFunctionBodyMissingLeftBraceProblem() { final ICompilerProblem lastError = Iterables.getLast(errors, null); if (lastError == null || lastError.getLine() < buffer.previous().getLine()) { final ICompilerProblem problem = new MissingLeftBraceBeforeFunctionBodyProblem(buffer.LT(1)); addProblem(problem); } } /** * Consume until one of the following tokens: * <ul> * <li>keyword</li> * <li>identifier</li> * <li>EOF</li> * <li>metadata tag</li> * <li>ASDoc comment</li> * <li>token type of the given exit condition</li> * </ul> * * @param exitCondition Stop if the next token type matches the exit * condition. */ protected void consumeUntilKeywordOrIdentifier(int exitCondition) { consumeUntilKeywordOr(ASTokenTypes.TOKEN_IDENTIFIER, ASTokenTypes.TOKEN_ATTRIBUTE, ASTokenTypes.TOKEN_ASDOC_COMMENT, exitCondition); } /** * Consume until a keyword, EOF or specified token types. * * @param types Stop if the next token's type is one of these. */ protected void consumeUntilKeywordOr(final Integer... types) { final ImmutableSet<Integer> stopSet = new ImmutableSet.Builder<Integer>().add(types).add(ASTokenTypes.EOF) .build(); while (!stopSet.contains(buffer.LA(1)) && !LT(1).isKeywordOrContextualReservedWord()) { consume(); } } @Override public final void match(final int t) throws MismatchedTokenException, TokenStreamException { final int la = LA(1); if (la != t) { exceptionPool.expecting = t; exceptionPool.token = LT(1); throw exceptionPool; } // Only update the "block depth" tracker when the parser is not in // syntactic predicates. if (inputState.guessing == 0) { switch (la) { case TOKEN_BLOCK_OPEN: blockDepth++; break; case TOKEN_BLOCK_CLOSE: blockDepth = blockDepth > 0 ? blockDepth - 1 : 0; break; default: // Ignore other tokens. break; } } consume(); } @Override public final int LA(final int i) { return buffer.LA(i); } @Override public final ASToken LT(final int i) { return buffer.LT(i); } @Override public final int mark() { return buffer.mark(); } @Override public final void rewind(final int pos) { buffer.rewind(pos); } protected static enum ExpressionMode { normal, noIn, e4x } /** * Binary precedence table used for expression parsing optimization. */ private static final ImmutableMap<Integer, Integer> BINARY_PRECEDENCE = new ImmutableMap.Builder<Integer, Integer>() .put(TOKEN_COMMA, 1).put(TOKEN_OPERATOR_ASSIGNMENT, 2).put(TOKEN_OPERATOR_LOGICAL_AND_ASSIGNMENT, 2) .put(TOKEN_OPERATOR_LOGICAL_OR_ASSIGNMENT, 2).put(TOKEN_OPERATOR_PLUS_ASSIGNMENT, 2) .put(TOKEN_OPERATOR_MINUS_ASSIGNMENT, 2).put(TOKEN_OPERATOR_MULTIPLICATION_ASSIGNMENT, 2) .put(TOKEN_OPERATOR_DIVISION_ASSIGNMENT, 2).put(TOKEN_OPERATOR_MODULO_ASSIGNMENT, 2) .put(TOKEN_OPERATOR_BITWISE_AND_ASSIGNMENT, 2).put(TOKEN_OPERATOR_BITWISE_OR_ASSIGNMENT, 2) .put(TOKEN_OPERATOR_BITWISE_XOR_ASSIGNMENT, 2).put(TOKEN_OPERATOR_BITWISE_LEFT_SHIFT_ASSIGNMENT, 2) .put(TOKEN_OPERATOR_BITWISE_RIGHT_SHIFT_ASSIGNMENT, 2) .put(TOKEN_OPERATOR_BITWISE_UNSIGNED_RIGHT_SHIFT_ASSIGNMENT, 2).put(TOKEN_OPERATOR_TERNARY, 3) .put(TOKEN_OPERATOR_LOGICAL_OR, 4).put(TOKEN_OPERATOR_LOGICAL_AND, 5).put(TOKEN_OPERATOR_BITWISE_OR, 6) .put(TOKEN_OPERATOR_BITWISE_XOR, 7).put(TOKEN_OPERATOR_BITWISE_AND, 8).put(TOKEN_OPERATOR_EQUAL, 9) .put(TOKEN_OPERATOR_NOT_EQUAL, 9).put(TOKEN_OPERATOR_STRICT_EQUAL, 9) .put(TOKEN_OPERATOR_STRICT_NOT_EQUAL, 9).put(TOKEN_OPERATOR_GREATER_THAN, 10) .put(TOKEN_OPERATOR_GREATER_THAN_EQUALS, 10).put(TOKEN_OPERATOR_LESS_THAN, 10) .put(TOKEN_OPERATOR_LESS_THAN_EQUALS, 10).put(TOKEN_KEYWORD_INSTANCEOF, 10).put(TOKEN_KEYWORD_IS, 10) .put(TOKEN_KEYWORD_AS, 10).put(TOKEN_KEYWORD_IN, 10).put(TOKEN_OPERATOR_BITWISE_LEFT_SHIFT, 11) .put(TOKEN_OPERATOR_BITWISE_RIGHT_SHIFT, 11).put(TOKEN_OPERATOR_BITWISE_UNSIGNED_RIGHT_SHIFT, 11) .put(TOKEN_OPERATOR_MINUS, 12).put(TOKEN_OPERATOR_PLUS, 12).put(TOKEN_OPERATOR_DIVISION, 13) .put(TOKEN_OPERATOR_MODULO, 13).put(TOKEN_OPERATOR_STAR, 13).build(); /** * Get the precedence of the given operator token. * * @param op Operator token. * @return Precedence of the operator token. */ private final int precedence(final ASToken op) { // When processing IN in a for-loop, drop precedence to 0, so that operator precedence parsing halts. if (expressionMode == ExpressionMode.noIn && op.getType() == TOKEN_KEYWORD_IN) return 0; final Integer precedence = BINARY_PRECEDENCE.get(op.getType()); if (precedence != null) return precedence; else return 0; } /** * Operator precedence parser. Refer to Vaughan Pratt, * "top down operator precedence parsing" or Dijkstra's shunting yard * algorithm, or precedence climbing. */ protected final ExpressionNodeBase precedenceParseExpression(int prec) throws RecognitionException, TokenStreamException { ExpressionNodeBase result = unaryExpr(); ASToken op = LT(1); int p1 = precedence(op); int p2 = p1; for (; p1 >= prec; p1--) { while (p2 == p1) { consume(); ExpressionNodeBase t = precedenceParseExpression(p1 + 1); // parse any sub expressions that are of higher precedence result = (ExpressionNodeBase) BinaryOperatorNodeBase.create(op, result, t); op = LT(1); p2 = precedence(op); } } return result; } /** * Expression mode of the current parsing state. {@code ASParser} updates * this field according to its context. */ protected ExpressionMode expressionMode = ExpressionMode.normal; /** * Provides method signature to {@link ASParser#unaryExpr()} so that * {@link #precedenceParseExpression()} can call it. */ abstract ExpressionNodeBase unaryExpr() throws RecognitionException, TokenStreamException; /** * True if a {@link ExtraCharactersAfterEndOfProgramProblem} was logged * already. */ private boolean extraCharacterAfterEndOfProgramProblemLogged = false; /** * Call this method when extra characters were found after the end of the * program. The {@link ExtraCharactersAfterEndOfProgramProblem} is only * logged when it's the first error in the current file. Even though this * method might be called multiple times because of the parser recovery * logic, such problem can only be reported once. * * @param token Token recognized as "extra characters". */ protected void foundExtraCharacterAfterEndOfProgram(ASToken token) { if (!extraCharacterAfterEndOfProgramProblemLogged && errors.isEmpty()) { extraCharacterAfterEndOfProgramProblemLogged = true; addProblem(new ExtraCharactersAfterEndOfProgramProblem(token)); } } /** * Call this method in the grammar once a "restricted token" is matched. * According to ECMA-262 Section 7.9.1: * <p> * <blockquote> When, as the program is parsed from left to right, a token * is encountered that is allowed by some production of the grammar, but the * production is a restricted production and the token would be the first * token for a terminal or nonterminal immediately following the annotation * [no LineTerminator here] within the restricted production (and therefore * such a token is called a restricted token), and the restricted token is * separated from the previous token by at least one LineTerminator, then a * semicolon is automatically inserted before the restricted token. * </blockquote> * <p> * These are the only restricted productions in ECMA grammar: * * <pre> * Expression [no LineTerminator] ++ * Expression [no LineTerminator] -- * continue [no LineTerminator] Identifier * break [no LineTerminator] Identifier * return [no LineTerminator] Expression * throw [no LineTerminator] Expression * </pre> * * As a result, this method should be called after these tokens are matched: * {@code continue, break, return, throw}. * * @param current The current token should be the restricted token. * @return True if the semicolon is inserted. * @note See ECMA 262 section 7.9.1 for details about semicolon insertion rules. */ protected boolean afterRestrictedToken(final ASToken current) { assert RESTRICTED_TOKEN_TYPES.contains(current.getType()) : "Unexpected restricted token type: " + current.getTypeString(); final ASToken nextToken = LT(1); // If the next token is a semicolon, even when there's a new-line, we // needn't insert a virtual semicolon. if (nextToken != null && nextToken.getLine() > current.getLine() && nextToken.getType() != TOKEN_SEMICOLON) return buffer.insertSemicolon(true); else return false; } /** * If any of these tokens is followed by a new line, we must insert a * semicolon after it. The set is only used by the assertion in * {@link #afterRestrictedToken(ASToken)}. */ private static final ImmutableSet<Integer> RESTRICTED_TOKEN_TYPES = ImmutableSet.of( ASTokenTypes.TOKEN_KEYWORD_RETURN, ASTokenTypes.TOKEN_KEYWORD_CONTINUE, ASTokenTypes.TOKEN_KEYWORD_BREAK, ASTokenTypes.TOKEN_KEYWORD_THROW); /** * See Javadoc for {@link #afterRestrictedToken(ASToken)} for details about * "restricted token" and semicolon insertion. * * @param nextToken The next token should be the restricted token. * @return True if the semicolon is inserted. * @see #afterRestrictedToken(ASToken) */ protected boolean beforeRestrictedToken(final ASToken nextToken) { // The next token should be the restricted token ++ or --. final ASToken current = buffer.previous(); assert nextToken != null : "token can't be null"; assert nextToken.getType() == ASTokenTypes.TOKEN_OPERATOR_INCREMENT || nextToken .getType() == ASTokenTypes.TOKEN_OPERATOR_DECREMENT : "Unexpected restricted token type: " + nextToken.getTypeString(); // If the next token is a semicolon, even when there's a new-line, we // needn't insert a virtual semicolon. if (nextToken != null && nextToken.getLine() > current.getLine() && nextToken.getType() != TOKEN_SEMICOLON) return buffer.insertSemicolon(false); else return false; } /** * Log an "unexpected token" problem. * * @param site offending token * @param expectedKind expected token kind * @return {@link UnexpectedTokenProblem} instance. */ protected final ICompilerProblem unexpectedTokenProblem(ASToken site, ASTokenKind expectedKind) { if (site.getType() == EOF) { // EOF token is synthesized. It doesn't have reasonable source location. // Use the last token instead. return new UnexpectedEOFProblem(buffer.previous(), expectedKind); } else { return new UnexpectedTokenProblem(site, expectedKind); } } /** * @see ASL syntax spec chapter 4.1 */ private static final ImmutableSet<String> RESERVED_NAMESPACE_ATTRIBUTES = ImmutableSet.of("private", "public", "protected", "internal"); /** * Make sure there's at most one namespace attribute found. Report syntax * problem otherwise. * * @param namespaceAttributes A list of namespace attributes. * @note See AS3 syntax specification chapter 8 - Enforcement of Syntactic * Restrictions. */ protected void verifyNamespaceAttributes(final List<INamespaceDecorationNode> namespaceAttributes) { if (namespaceAttributes.size() < 2) return; final List<INamespaceDecorationNode> reservedNamespaceAttributes = new ArrayList<INamespaceDecorationNode>(); final List<INamespaceDecorationNode> userDefinedNamespaceAttributes = new ArrayList<INamespaceDecorationNode>(); for (final INamespaceDecorationNode namespaceAttribute : namespaceAttributes) { final String namespaceName = namespaceAttribute.getName(); if (RESERVED_NAMESPACE_ATTRIBUTES.contains(namespaceName)) reservedNamespaceAttributes.add(namespaceAttribute); else userDefinedNamespaceAttributes.add(namespaceAttribute); } if (reservedNamespaceAttributes.size() > 1) { final MultipleReservedNamespaceAttributesProblem problem = new MultipleReservedNamespaceAttributesProblem( reservedNamespaceAttributes.get(0)); addProblem(problem); } if (userDefinedNamespaceAttributes.size() > 1) { final MultipleNamespaceAttributesProblem problem = new MultipleNamespaceAttributesProblem( userDefinedNamespaceAttributes.get(0)); addProblem(problem); } if (!userDefinedNamespaceAttributes.isEmpty() && !reservedNamespaceAttributes.isEmpty()) { final NamespaceAttributeNotAllowedProblem problem = new NamespaceAttributeNotAllowedProblem( userDefinedNamespaceAttributes.get(0)); addProblem(problem); } } private static final ImmutableSet<String> CONTEXTUAL_RESERVED_KEYWORD_MODIFIERS = ImmutableSet.of("dynamic", "final", "native", "static", "override"); /** * Recover from {@link CanNotInsertSemicolonProblem} after an expression * statement. Such error pattern is usually a sign of invalid code being * parsed as expressions. * <p> * <h2>Recover from contextual keywords</h2> When there's syntax errors, the * token transformation logic in {@link StreamingASTokenizer} might fail. * For example, a contextual reserved keyword like "static" is labeled as * "identifier" instead of "modifier" when the token following "static" * doesn't start a definition. As a result "static" is parsed as a single * variable "expressionStatement", and the following content will be * reported with a {@link CanNotInsertSemicolonProblem}. * </p> * <p> * Since we want the error reporting to be user-friendly, we need to detect * such cases and override the compiler problems. The pattern is simple: an * {@link ExpressionNodeBase} of type {@link IdentifierNode} and it's * identifier name is one of the contextual reserved keywords: * {@code dynamic}, {@code final}, {@code native}, {@code static} and * {@code override}. * </p> * <p> * <h2>Recover from invalid labels</h2> When the next offending token is * colon, it means the parsed expression isn't a valid label identifier. * </p> * * @param e The AST created for the parsed expression. */ protected final void recoverFromExpressionStatementMissingSemicolon(final ExpressionNodeBase e) { final ASToken lt = buffer.LT(1); ICompilerProblem problem = null; if (e instanceof IdentifierNode) { final String name = ((IdentifierNode) e).getName(); if (CONTEXTUAL_RESERVED_KEYWORD_MODIFIERS.contains(name)) { if (lt.getType() == ASTokenTypes.TOKEN_KEYWORD_PACKAGE) { problem = new AttributesNotAllowedOnPackageDefinitionProblem(lt); } else { problem = new ExpectDefinitionKeywordAfterAttributeProblem(name, lt); } } } else if (lt.getType() == TOKEN_COLON) { problem = new InvalidLabelProblem(lt); consume(); // drop the colon } if (problem != null) { // replace the syntax problem final ICompilerProblem lastError = Iterables.getLast(errors); errors.remove(lastError); errors.add(problem); } } /** * Traps erroneous syntax like: * * <pre> * innerLabel: namespace ns1; * </pre> * * Although the {@link InvalidAttributeProblem} doesn't make much sense in * this context, we are just being compatible with the old ASC's behavior. * * @param offendingNSToken The offending "namespace" token after the label * and colon. */ protected void trapInvalidSubstatement(ASToken offendingNSToken) { final InvalidAttributeProblem problem = new InvalidAttributeProblem(offendingNSToken); addProblem(problem); } /** * Traps erroneous syntax like: * * <pre> * package foo * { * ns1 private class T {} * } * </pre> * * @param offendingNSToken The unexpected namespace name token. */ protected void trapInvalidNamespaceAttribute(ASToken offendingNSToken) { final NamespaceAttributeNotAllowedProblem problem = new NamespaceAttributeNotAllowedProblem( offendingNSToken); addProblem(problem); } /** * Evaluate configuration variable result. * * @param configNamespace The configuration namespace. * @param opToken The operator token. Always "::". * @param configVar The unqualified configuration variable name. * @return True if the qualified configuration variable evaluates to true. */ protected boolean evaluateConfigurationVariable(final String configNamespace, final ASToken opToken, final String configVar) { final ConfigExpressionNode configExpression = new ConfigExpressionNode( new NamespaceIdentifierNode(configNamespace), opToken, new IdentifierNode(configVar)); final Object value = configProcessor.evaluateConstNodeExpressionToJavaObject(configExpression); return value == null ? false : ECMASupport.toBoolean(value); } /** * Report unexpected token problem. * * @param token Offending token. */ protected final void reportUnexpectedTokenProblem(final ASToken token) { addProblem(new SyntaxProblem(token)); } /** * Consume all the tokens in a function body, excluding the open curly and * close curly braces. * <p> * In order to allow lazy-parsing of the skipped function bodies, the * contents need to be cached. We can't cache all the tokens because they * take up huge amount of memory. * <p> * On the other hand, we can't cache the text of the function body by * accessing the underlying {@code Reader} directly, because it isn't * guaranteed to point to the start of the function block when then the * parser sees the function body due to the possible "look-ahead" operations * triggered by the tokenizer. * <p> * As a result, it leaves us the only option which is to capture the start * and end character offsets of the function body. * <p> * Since {@link Reader#skip(long)} is slow, the parser keeps a secondary * {@code Reader} ({@link #secondaryReader}) in order to cache the text of * the function body on-the-fly. The text is stored on the corresponding * {@link FunctionNode}. * <p> * This feature is turn on only <b>all</b> of the following conditions are * true: * <ul> * <li>The source is from a readable ActionScript source file.</li> * <li>The function is not a function closure.</li> * <li>The source file is not an in-memory compilation (see * {@link IInvisibleCompilationUnit}) unit in the editor.</li> * </ul> * * @param functionNode Function node. * @param openT The open curly token of the function body. * @see <a * href="https://zerowing.corp.adobe.com/display/compiler/Deferring+function+body+nodes">Deferring * function body nodes</a> */ protected final void skipFunctionBody(final FunctionNode functionNode, final ASToken openT) { assert functionNode != null : "Function node can't be null."; assert openT != null && openT.getType() == TOKEN_BLOCK_OPEN : "Expected '{' token"; // this feature is not applicable to this function node if (deferFunctionBody != DeferFunctionBody.ENABLED || functionNode.getParent() instanceof FunctionObjectNode) return; // If the first token in the function body is "}" or "EOF", the function // body is empty. There's nothing to skip. if (LA(1) == TOKEN_BLOCK_CLOSE || LA(1) == EOF) return; // Start skipping function body by matching curly braces. boolean functionBodyHasInclude = false; ASToken prevToken = null; tokenLoop: for (int depth = 0; depth > 0 || LA(1) != TOKEN_BLOCK_CLOSE; consume()) { prevToken = LT(1); // If a function body token's source path is different from the // function node's source path, there's include processing. Then, // the function body text caching optimization can't be used. if (!this.getSourceFilePath().equals(LT(1).getSourcePath())) functionBodyHasInclude = true; switch (prevToken.getType()) { case TOKEN_BLOCK_OPEN: depth++; break; case TOKEN_BLOCK_CLOSE: depth--; break; case EOF: break tokenLoop; } } assert LA(1) == TOKEN_BLOCK_CLOSE || LA(1) == EOF : "Loop should stop before the '}' of the function body or 'eof'."; assert prevToken != null : "Function body must have at least one token if we reached here."; final StringBuilder functionBodyText = tryGetFunctionBodyText(openT, functionBodyHasInclude, prevToken); functionNode.setFunctionBodyInfo(openT, prevToken, configProcessor, functionBodyText); fileNodeAccumulator.addDeferredFunctionNode(functionNode); } /** * Optimization: cache function body text if there's no include processing. * * @param openT Open curly token of the function body. * @param functionBodyHasInclude True if there are "include" directives in * the function body. * @param lastToken Last token of the function body (excluding close curly * "}"). * @return Function body text or null. */ private StringBuilder tryGetFunctionBodyText(final ASToken openT, final boolean functionBodyHasInclude, final ASToken lastToken) { final StringBuilder functionBodyText; if (secondaryReader != null && !functionBodyHasInclude) { // Get function body text functionBodyText = new StringBuilder(); try { // Move secondary reader to the beginning of a function body. final int readerSkip = openT.getLocalEnd() - secondaryReaderPosition; assert readerSkip >= 0 : "Invalid position"; final long skip = secondaryReader.skip(readerSkip); assert skip == readerSkip : "The buffer didn't skip full length."; // Read text till the end of the function body (including the // closing curly if there is one). final ASToken endToken; if (LA(1) == TOKEN_BLOCK_CLOSE) endToken = LT(1); else endToken = lastToken; for (int position = openT.getLocalEnd(); position < endToken.getLocalEnd(); position++) { functionBodyText.append((char) secondaryReader.read()); } // Update secondary reader position to the end of the function body. secondaryReaderPosition = endToken.getLocalEnd(); } catch (IOException e) { String path = openT.getSourcePath(); ICompilerProblem problem = (path == null) ? new InternalCompilerProblem(e) : new InternalCompilerProblem2(path, e, SUB_SYSTEM); errors.add(problem); } } else { functionBodyText = null; } return functionBodyText; } /** * Increment package depth counter and check for * {@link NestedPackageProblem}. * * @param keywordPackage "package" keyword token. */ protected final void enterPackage(ASToken keywordPackage) { if (packageDepth > 0) { final ICompilerProblem problem = new NestedPackageProblem(keywordPackage); addProblem(problem); } else if (!isGlobalContext()) { final ICompilerProblem problem = new SyntaxProblem(keywordPackage); addProblem(problem); } packageDepth++; } /** * Called when the parser enters a type application expression. It validates * the "collection expression" of the type application expression. It must * be either a {@code MemberAccessExpressionNode} or an * {@code IdentifierNode}. * * @param collectionExpression The expression node on the left-hand side of * {@code .< >}. */ protected final void enterTypeApplication(final ExpressionNodeBase collectionExpression) { if (collectionExpression == null || collectionExpression.getNodeID() == ASTNodeID.IdentifierID || collectionExpression.getNodeID() == ASTNodeID.FunctionCallID || collectionExpression.getNodeID() == ASTNodeID.TypedExpressionID || collectionExpression instanceof MemberAccessExpressionNode) { return; } else { final ICompilerProblem problem = new InvalidTypeProblem(LT(1), collectionExpression.getNodeID().getParaphrase()); addProblem(problem); } } /** * Enter a "group" and increment the depth counter. */ protected final void enterGroup() { groupDepth++; } /** * Leave a "group" and decrement the depth counter. */ protected final void leaveGroup() { groupDepth--; } /** * Check if class definition is allowed. * * @param keywordClass {@code class} keyword token. */ protected final void enterClassDefinition(final ASToken keywordClass) { if (!isGlobalContext()) addProblem(new NestedClassProblem(keywordClass)); } /** * Check if interface definition is allowed. * * @param keywordInterface {@code interface} keyword token. */ protected final void enterInterfaceDefinition(final ASToken keywordInterface) { if (!isGlobalContext()) addProblem(new NestedInterfaceProblem(keywordInterface)); } /** * Decrement package depth counter. */ protected final void leavePackage() { assert blockDepth >= 0 : "package depth should never be negative"; packageDepth = packageDepth > 0 ? packageDepth - 1 : 0; } /** * A syntax tree is in a global context if it is nested by a <i>program</i> * without crossing a <i>block</i>, or nested by the <i>block</i> of a * <i>package directive</i> without crossing another <i>block</i>. * <p> * Note that directive-level blocks are called "groups". Entering a group * doesn't cause the parser to leave global context. * * @return True if the current context is global context. */ protected final boolean isGlobalContext() { assert blockDepth >= 0 : "block depth should never be negative"; assert packageDepth <= blockDepth : "package depth can't be greater than block depth"; assert groupDepth <= blockDepth : "group depth can't be greater than block depth"; assert packageDepth + groupDepth <= blockDepth : "invalid state of block depth"; return blockDepth - packageDepth - groupDepth == 0; } /** * Disambiguate between a function definition and a function closure. * * @return True if the next "function" keyword defines a closure. */ protected final boolean isFunctionClosure() { return LA(1) == TOKEN_KEYWORD_FUNCTION && (LA(2) == TOKEN_PAREN_OPEN || buffer.previous().canPreceedAnonymousFunction()); } /** * Ignore missing semicolons in the following two cases: * * <pre> * do x++ while (x<10); // after x++ * if (x<10) x++ else x--; // after x++ * </pre> */ protected final void afterInnerSubstatement() { final ICompilerProblem lastError = Iterables.getLast(errors, null); if (lastError != null && lastError instanceof CanNotInsertSemicolonProblem) { // Ideally, we should compare absolute offset. However, CMP-1828 requires // all syntax problems to have local offset. final int problemEnd = ((CanNotInsertSemicolonProblem) lastError).getEnd(); final int lastTokenEnd = LT(1).getLocalEnd(); if (problemEnd == lastTokenEnd) errors.remove(lastError); } } /** * Recover from metadata parsing failure. Code model require error recovery * logic for the following case: * * <pre> * [ * function hello():void {} * </pre> * * The incomplete square bracket is expected to be matched as part of a * metadata tag, and the following function definition is properly built. * However, function objects in array literals are valid: * * <pre> * var fs = [ function f1():void{}, function f2():void{} ]; * </pre> * * In order to appease code hinting, the recovery logic will hoist the * elements in the failed array literal into the parent node of the array * literal node. * * @param container Parent node of array literal node. * @param arrayLiteralNode Array literal node, whose contents will be * hoisted. */ protected final void recoverFromMetadataTag(final ContainerNode container, final ArrayLiteralNode arrayLiteralNode) { if (container == null || arrayLiteralNode == null) return; // Hoist content associated with the failed array literal. final ContainerNode contents = arrayLiteralNode.getContentsNode(); for (int i = 0; i < contents.getChildCount(); i++) { final IASNode child = contents.getChild(i); if (child.getNodeID() == ASTNodeID.FunctionObjectID) container.addChild(((FunctionObjectNode) child).getFunctionNode()); else container.addChild((NodeBase) child); } // Handle special case: // [ // var x:int; // The keyword 'var' is turned into a identifier by 'handleParsingError()'. final ASToken lookback = buffer.previous(); if (lookback != null && lookback.getType() == TOKEN_IDENTIFIER && lookback.getText().equals(IASKeywordConstants.VAR)) { lookback.setType(TOKEN_KEYWORD_VAR); buffer.rewind(buffer.mark() - 1); } } /** * Determine if the parser should deploy an error trap for a * typing-in-progress metadata tag on a definition item. For example: * * <pre> * class T * { * [ * public var name:String; * * [ * public override function hello():void {} * * [ * function process(); * } * </pre> * * The error trap is enabled when the "[" token is the only token on the * current line, and the next token can start a definition item. */ protected final boolean isIncompleteMetadataTagOnDefinition() { if (LA(1) == TOKEN_SQUARE_OPEN && LT(1).getLine() > buffer.previous().getLine() && LT(2).getLine() > LT(1).getLine()) { // Note that function object is allowed inside array literal. switch (LA(2)) { case TOKEN_KEYWORD_VAR: case TOKEN_KEYWORD_CLASS: case TOKEN_KEYWORD_INTERFACE: case TOKEN_NAMESPACE_ANNOTATION: case TOKEN_MODIFIER_DYNAMIC: case TOKEN_MODIFIER_FINAL: case TOKEN_MODIFIER_NATIVE: case TOKEN_MODIFIER_OVERRIDE: case TOKEN_MODIFIER_STATIC: case TOKEN_MODIFIER_VIRTUAL: return true; } } return false; } /** * All tags with binding names such as {@code < foo}>} use this name in the * XML tag stack. */ private static final String TAG_NAME_BINDING = "{...}"; /** * Called when a {@code <foo} token is encountered. * * @param token {@code TOKEN_E4X_OPEN_TAG_START} token. */ protected final void xmlTagOpen(final ASToken token) { assert token.getType() == TOKEN_E4X_OPEN_TAG_START : "unexpected token type: " + token; assert token.getText().startsWith("<") : "unexpected token text: " + token; final String tagName = token.getText().substring(1); xmlTags.push(new XMLTagInfo(tagName, token)); } /** * Called when a {@code <} token is encountered. * * @param token {@code HIDDEN_TOKEN_E4X} token. */ protected final void xmlTagOpenBinding(final ASToken token) { assert token.getType() == HIDDEN_TOKEN_E4X : "unexpected token " + token; assert token.getText().equals("<") : "unexpected token text: " + token; xmlTags.push(new XMLTagInfo(TAG_NAME_BINDING, token)); } /** * Called when a {@code </foo} token is encountered. * * @param token {@code TOKEN_E4X_CLOSE_TAG_START} token. */ protected final void xmlTagClose(final ASToken token) { assert token.getType() == TOKEN_E4X_CLOSE_TAG_START : "unexpected token " + token; assert token.getText().startsWith("</") : "unexpected token text: " + token; final String tagName = token.getText().substring(2); if (!xmlTags.isEmpty()) { final XMLTagInfo openTag = xmlTags.pop(); if (TAG_NAME_BINDING.equals(openTag.name) || tagName.isEmpty()) { // At least one of the open or close tag's name is a binding. // No need to check if tags match. return; } else if (openTag.name.equals(tagName)) { // Open and close tags match. return; } else { addProblem(new XMLOpenCloseTagNotMatchProblem(token)); return; } } else { addProblem(new SyntaxProblem(token)); } } /** * Called when a {@code />} token is encountered. * * @param token {@code TOKEN_E4X_EMPTY_TAG_END} token. */ protected final void xmlEmptyTagEnd(final ASToken token) { assert token.getType() == TOKEN_E4X_EMPTY_TAG_END : "unexpected token " + token; assert token.getText().endsWith("/>") : "unexpected token text: " + token; if (!xmlTags.isEmpty()) { xmlTags.pop(); } } /** * Called when start parsing an XML literal. */ protected final void enterXMLLiteral() { assert xmlTags.isEmpty() : "Invalid state: must clear tag stack after each XML literal."; xmlTags.clear(); } /** * Called when finish parsing an XML literal. This method validates the XML. */ protected final void leaveXMLLiteral() { while (!xmlTags.isEmpty()) { final XMLTagInfo openTag = xmlTags.pop(); final ICompilerProblem problem = new XMLOpenCloseTagNotMatchProblem(openTag.location); addProblem(problem); } } /** * A tuple of XML tag name and its source location. */ private static final class XMLTagInfo { /** * XML tag name. */ final String name; /** * XML tag location. */ final ASToken location; XMLTagInfo(final String name, final ASToken location) { this.name = name; this.location = location; } } /** * Syntactic predicate for CMP-335. * * @return True if the next token is "public" namespace token. */ protected final boolean isNextTokenPublicNamespace() { final ASToken token = LT(1); return TOKEN_NAMESPACE_ANNOTATION == token.getType() && IASKeywordConstants.PUBLIC.equals(token.getText()); } /** * Check if the next attribute definition is gated with a configuration * condition variable. If it is, then return the evaluated result of the * condition. * * @param containerNode parent node * @return True if the attributed definition is enabled. */ protected final boolean isDefinitionEnabled(final ContainerNode containerNode) { assert containerNode != null : "Container node can't be null."; final int childCount = containerNode.getChildCount(); if (childCount > 0) { final IASNode lastChild = containerNode.getChild(childCount - 1); if (lastChild.getNodeID() == ASTNodeID.LiteralBooleanID) { // evaluate final boolean eval = Boolean.parseBoolean(((LiteralNode) lastChild).getValue()); // remove the configuration condition node containerNode.removeItem((NodeBase) lastChild); return eval; } } return true; } /** * Parses an ActionScript3 file and populate the FileNode. This method * forces the parser to retry after * {@code fileLevelDirectives(ContainerNode)} fails. Before retrying, the * parser makes sure one and only one * {@link ExtraCharactersAfterEndOfProgramProblem} is logged. Then, it * discards the offending token and continue matching at "directive" level. * <p> * I didn't write this as a regular ANTLR rule because the ANTLR-generated * code wouldn't enter {@code fileLevelDirectives(ContainerNode)} if the * lookahead is not one of the tokens in the "start set". */ public final void file(ContainerNode c) throws RecognitionException, TokenStreamException { while (LA(1) != EOF) { fileLevelDirectives(c); if (LA(1) != EOF) { foundExtraCharacterAfterEndOfProgram(LT(1)); consume(); } } } /** * @return true when we are parsing project vars (like the command line) */ public final boolean isParsingProjectConfigVariables() { return parsingProjectConfigVariables; } }