Java tutorial
/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.lucene.expressions.js; import java.io.IOException; import java.io.Reader; import java.lang.invoke.MethodHandles; import java.lang.invoke.MethodType; import java.lang.reflect.Constructor; import java.lang.reflect.Method; import java.lang.reflect.Modifier; import java.nio.charset.StandardCharsets; import java.text.ParseException; import java.util.ArrayDeque; import java.util.Arrays; import java.util.Collections; import java.util.Deque; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.Map; import java.util.Properties; import org.antlr.v4.runtime.ANTLRInputStream; import org.antlr.v4.runtime.CommonTokenStream; import org.antlr.v4.runtime.tree.ParseTree; import org.apache.lucene.expressions.Expression; import org.apache.lucene.expressions.js.JavascriptParser.ExpressionContext; import org.apache.lucene.search.DoubleValues; import org.apache.lucene.util.IOUtils; import org.objectweb.asm.ClassWriter; import org.objectweb.asm.Label; import org.objectweb.asm.Opcodes; import org.objectweb.asm.Type; import org.objectweb.asm.commons.GeneratorAdapter; /** * An expression compiler for javascript expressions. * <p> * Example: * <pre class="prettyprint"> * Expression foo = JavascriptCompiler.compile("((0.3*popularity)/10.0)+(0.7*score)"); * </pre> * <p> * See the {@link org.apache.lucene.expressions.js package documentation} for * the supported syntax and default functions. * <p> * You can compile with an alternate set of functions via {@link #compile(String, Map, ClassLoader)}. * For example: * <pre class="prettyprint"> * Map<String,Method> functions = new HashMap<>(); * // add all the default functions * functions.putAll(JavascriptCompiler.DEFAULT_FUNCTIONS); * // add cbrt() * functions.put("cbrt", Math.class.getMethod("cbrt", double.class)); * // call compile with customized function map * Expression foo = JavascriptCompiler.compile("cbrt(score)+ln(popularity)", * functions, * getClass().getClassLoader()); * </pre> * * @lucene.experimental */ public final class JavascriptCompiler { static final class Loader extends ClassLoader { Loader(ClassLoader parent) { super(parent); } public Class<? extends Expression> define(String className, byte[] bytecode) { return defineClass(className, bytecode, 0, bytecode.length).asSubclass(Expression.class); } } private static final int CLASSFILE_VERSION = Opcodes.V1_8; // We use the same class name for all generated classes as they all have their own class loader. // The source code is displayed as "source file name" in stack trace. private static final String COMPILED_EXPRESSION_CLASS = JavascriptCompiler.class.getName() + "$CompiledExpression"; private static final String COMPILED_EXPRESSION_INTERNAL = COMPILED_EXPRESSION_CLASS.replace('.', '/'); static final Type EXPRESSION_TYPE = Type.getType(Expression.class); static final Type FUNCTION_VALUES_TYPE = Type.getType(DoubleValues.class); private static final org.objectweb.asm.commons.Method EXPRESSION_CTOR = getAsmMethod(void.class, "<init>", String.class, String[].class), EVALUATE_METHOD = getAsmMethod(double.class, "evaluate", DoubleValues[].class); static final org.objectweb.asm.commons.Method DOUBLE_VAL_METHOD = getAsmMethod(double.class, "doubleValue"); /** create an ASM Method object from return type, method name, and parameters. */ private static org.objectweb.asm.commons.Method getAsmMethod(Class<?> rtype, String name, Class<?>... ptypes) { return new org.objectweb.asm.commons.Method(name, MethodType.methodType(rtype, ptypes).toMethodDescriptorString()); } // This maximum length is theoretically 65535 bytes, but as it's CESU-8 encoded we dont know how large it is in bytes, so be safe // rcmuir: "If your ranking function is that large you need to check yourself into a mental institution!" private static final int MAX_SOURCE_LENGTH = 16384; final String sourceText; final Map<String, Method> functions; /** * Compiles the given expression. * * @param sourceText The expression to compile * @return A new compiled expression * @throws ParseException on failure to compile */ public static Expression compile(String sourceText) throws ParseException { return new JavascriptCompiler(sourceText).compileExpression(JavascriptCompiler.class.getClassLoader()); } /** * Compiles the given expression with the supplied custom functions. * <p> * Functions must be {@code public static}, return {@code double} and * can take from zero to 256 {@code double} parameters. * * @param sourceText The expression to compile * @param functions map of String names to functions * @param parent a {@code ClassLoader} that should be used as the parent of the loaded class. * It must contain all classes referred to by the given {@code functions}. * @return A new compiled expression * @throws ParseException on failure to compile */ public static Expression compile(String sourceText, Map<String, Method> functions, ClassLoader parent) throws ParseException { if (parent == null) { throw new NullPointerException("A parent ClassLoader must be given."); } for (Method m : functions.values()) { checkFunctionClassLoader(m, parent); checkFunction(m); } return new JavascriptCompiler(sourceText, functions).compileExpression(parent); } /** * This method is unused, it is just here to make sure that the function signatures don't change. * If this method fails to compile, you also have to change the byte code generator to correctly * use the FunctionValues class. */ @SuppressWarnings({ "unused", "null" }) private static void unusedTestCompile() throws IOException { DoubleValues f = null; double ret = f.doubleValue(); } /** * Constructs a compiler for expressions. * @param sourceText The expression to compile */ private JavascriptCompiler(String sourceText) { this(sourceText, DEFAULT_FUNCTIONS); } /** * Constructs a compiler for expressions with specific set of functions * @param sourceText The expression to compile */ private JavascriptCompiler(String sourceText, Map<String, Method> functions) { if (sourceText == null) { throw new NullPointerException(); } this.sourceText = sourceText; this.functions = functions; } /** * Compiles the given expression with the specified parent classloader * * @return A new compiled expression * @throws ParseException on failure to compile */ private Expression compileExpression(ClassLoader parent) throws ParseException { final Map<String, Integer> externalsMap = new LinkedHashMap<>(); final ClassWriter classWriter = new ClassWriter(ClassWriter.COMPUTE_FRAMES | ClassWriter.COMPUTE_MAXS); try { generateClass(getAntlrParseTree(), classWriter, externalsMap); final Class<? extends Expression> evaluatorClass = new Loader(parent).define(COMPILED_EXPRESSION_CLASS, classWriter.toByteArray()); final Constructor<? extends Expression> constructor = evaluatorClass.getConstructor(String.class, String[].class); return constructor.newInstance(sourceText, externalsMap.keySet().toArray(new String[externalsMap.size()])); } catch (RuntimeException re) { if (re.getCause() instanceof ParseException) { throw (ParseException) re.getCause(); } throw re; } catch (ReflectiveOperationException exception) { throw new IllegalStateException( "An internal error occurred attempting to compile the expression (" + sourceText + ").", exception); } } /** * Parses the sourceText into an ANTLR 4 parse tree * * @return The ANTLR parse tree * @throws ParseException on failure to parse */ private ParseTree getAntlrParseTree() throws ParseException { final ANTLRInputStream antlrInputStream = new ANTLRInputStream(sourceText); final JavascriptErrorHandlingLexer javascriptLexer = new JavascriptErrorHandlingLexer(antlrInputStream); javascriptLexer.removeErrorListeners(); final JavascriptParser javascriptParser = new JavascriptParser(new CommonTokenStream(javascriptLexer)); javascriptParser.removeErrorListeners(); javascriptParser.setErrorHandler(new JavascriptParserErrorStrategy()); return javascriptParser.compile(); } /** * Sends the bytecode of class file to {@link ClassWriter}. */ private void generateClass(final ParseTree parseTree, final ClassWriter classWriter, final Map<String, Integer> externalsMap) throws ParseException { classWriter.visit(CLASSFILE_VERSION, Opcodes.ACC_PUBLIC | Opcodes.ACC_SUPER | Opcodes.ACC_FINAL, COMPILED_EXPRESSION_INTERNAL, null, EXPRESSION_TYPE.getInternalName(), null); final String clippedSourceText = (sourceText.length() <= MAX_SOURCE_LENGTH) ? sourceText : (sourceText.substring(0, MAX_SOURCE_LENGTH - 3) + "..."); classWriter.visitSource(clippedSourceText, null); final GeneratorAdapter constructor = new GeneratorAdapter(Opcodes.ACC_PUBLIC, EXPRESSION_CTOR, null, null, classWriter); constructor.loadThis(); constructor.loadArgs(); constructor.invokeConstructor(EXPRESSION_TYPE, EXPRESSION_CTOR); constructor.returnValue(); constructor.endMethod(); final GeneratorAdapter gen = new GeneratorAdapter(Opcodes.ACC_PUBLIC, EVALUATE_METHOD, null, null, classWriter); // to completely hide the ANTLR visitor we use an anonymous impl: new JavascriptBaseVisitor<Void>() { private final Deque<Type> typeStack = new ArrayDeque<>(); @Override public Void visitCompile(JavascriptParser.CompileContext ctx) { typeStack.push(Type.DOUBLE_TYPE); visit(ctx.expression()); typeStack.pop(); return null; } @Override public Void visitPrecedence(JavascriptParser.PrecedenceContext ctx) { visit(ctx.expression()); return null; } @Override public Void visitNumeric(JavascriptParser.NumericContext ctx) { if (ctx.HEX() != null) { pushLong(Long.parseLong(ctx.HEX().getText().substring(2), 16)); } else if (ctx.OCTAL() != null) { pushLong(Long.parseLong(ctx.OCTAL().getText().substring(1), 8)); } else if (ctx.DECIMAL() != null) { gen.push(Double.parseDouble(ctx.DECIMAL().getText())); gen.cast(Type.DOUBLE_TYPE, typeStack.peek()); } else { throw new IllegalStateException("Unknown operation specified: " + ctx.getText()); } return null; } @Override public Void visitExternal(JavascriptParser.ExternalContext ctx) { String text = ctx.VARIABLE().getText(); int arguments = ctx.expression().size(); boolean parens = ctx.LP() != null && ctx.RP() != null; Method method = parens ? functions.get(text) : null; try { if (method != null) { int arity = method.getParameterTypes().length; if (arguments != arity) { throw new ParseException("Invalid expression '" + sourceText + "': Expected (" + arity + ") arguments for function call (" + text + "), but found (" + arguments + ").", ctx.start.getStartIndex()); } typeStack.push(Type.DOUBLE_TYPE); for (int argument = 0; argument < arguments; ++argument) { visit(ctx.expression(argument)); } typeStack.pop(); gen.invokeStatic(Type.getType(method.getDeclaringClass()), org.objectweb.asm.commons.Method.getMethod(method)); gen.cast(Type.DOUBLE_TYPE, typeStack.peek()); } else if (!parens || arguments == 0 && text.contains(".")) { int index; text = normalizeQuotes(ctx.getText()); if (externalsMap.containsKey(text)) { index = externalsMap.get(text); } else { index = externalsMap.size(); externalsMap.put(text, index); } gen.loadArg(0); gen.push(index); gen.arrayLoad(FUNCTION_VALUES_TYPE); gen.invokeVirtual(FUNCTION_VALUES_TYPE, DOUBLE_VAL_METHOD); gen.cast(Type.DOUBLE_TYPE, typeStack.peek()); } else { throw new ParseException("Invalid expression '" + sourceText + "': Unrecognized function call (" + text + ").", ctx.start.getStartIndex()); } return null; } catch (ParseException e) { // The API doesn't allow checked exceptions here, so propagate up the stack. This is unwrapped // in getAntlrParseTree. throw new RuntimeException(e); } } @Override public Void visitUnary(JavascriptParser.UnaryContext ctx) { if (ctx.BOOLNOT() != null) { Label labelNotTrue = new Label(); Label labelNotReturn = new Label(); typeStack.push(Type.INT_TYPE); visit(ctx.expression()); typeStack.pop(); gen.visitJumpInsn(Opcodes.IFEQ, labelNotTrue); pushBoolean(false); gen.goTo(labelNotReturn); gen.visitLabel(labelNotTrue); pushBoolean(true); gen.visitLabel(labelNotReturn); } else if (ctx.BWNOT() != null) { typeStack.push(Type.LONG_TYPE); visit(ctx.expression()); typeStack.pop(); gen.push(-1L); gen.visitInsn(Opcodes.LXOR); gen.cast(Type.LONG_TYPE, typeStack.peek()); } else if (ctx.ADD() != null) { visit(ctx.expression()); } else if (ctx.SUB() != null) { typeStack.push(Type.DOUBLE_TYPE); visit(ctx.expression()); typeStack.pop(); gen.visitInsn(Opcodes.DNEG); gen.cast(Type.DOUBLE_TYPE, typeStack.peek()); } else { throw new IllegalStateException("Unknown operation specified: " + ctx.getText()); } return null; } @Override public Void visitMuldiv(JavascriptParser.MuldivContext ctx) { int opcode; if (ctx.MUL() != null) { opcode = Opcodes.DMUL; } else if (ctx.DIV() != null) { opcode = Opcodes.DDIV; } else if (ctx.REM() != null) { opcode = Opcodes.DREM; } else { throw new IllegalStateException("Unknown operation specified: " + ctx.getText()); } pushArith(opcode, ctx.expression(0), ctx.expression(1)); return null; } @Override public Void visitAddsub(JavascriptParser.AddsubContext ctx) { int opcode; if (ctx.ADD() != null) { opcode = Opcodes.DADD; } else if (ctx.SUB() != null) { opcode = Opcodes.DSUB; } else { throw new IllegalStateException("Unknown operation specified: " + ctx.getText()); } pushArith(opcode, ctx.expression(0), ctx.expression(1)); return null; } @Override public Void visitBwshift(JavascriptParser.BwshiftContext ctx) { int opcode; if (ctx.LSH() != null) { opcode = Opcodes.LSHL; } else if (ctx.RSH() != null) { opcode = Opcodes.LSHR; } else if (ctx.USH() != null) { opcode = Opcodes.LUSHR; } else { throw new IllegalStateException("Unknown operation specified: " + ctx.getText()); } pushShift(opcode, ctx.expression(0), ctx.expression(1)); return null; } @Override public Void visitBoolcomp(JavascriptParser.BoolcompContext ctx) { int opcode; if (ctx.LT() != null) { opcode = GeneratorAdapter.LT; } else if (ctx.LTE() != null) { opcode = GeneratorAdapter.LE; } else if (ctx.GT() != null) { opcode = GeneratorAdapter.GT; } else if (ctx.GTE() != null) { opcode = GeneratorAdapter.GE; } else { throw new IllegalStateException("Unknown operation specified: " + ctx.getText()); } pushCond(opcode, ctx.expression(0), ctx.expression(1)); return null; } @Override public Void visitBooleqne(JavascriptParser.BooleqneContext ctx) { int opcode; if (ctx.EQ() != null) { opcode = GeneratorAdapter.EQ; } else if (ctx.NE() != null) { opcode = GeneratorAdapter.NE; } else { throw new IllegalStateException("Unknown operation specified: " + ctx.getText()); } pushCond(opcode, ctx.expression(0), ctx.expression(1)); return null; } @Override public Void visitBwand(JavascriptParser.BwandContext ctx) { pushBitwise(Opcodes.LAND, ctx.expression(0), ctx.expression(1)); return null; } @Override public Void visitBwxor(JavascriptParser.BwxorContext ctx) { pushBitwise(Opcodes.LXOR, ctx.expression(0), ctx.expression(1)); return null; } @Override public Void visitBwor(JavascriptParser.BworContext ctx) { pushBitwise(Opcodes.LOR, ctx.expression(0), ctx.expression(1)); return null; } @Override public Void visitBooland(JavascriptParser.BoolandContext ctx) { Label andFalse = new Label(); Label andEnd = new Label(); typeStack.push(Type.INT_TYPE); visit(ctx.expression(0)); gen.visitJumpInsn(Opcodes.IFEQ, andFalse); visit(ctx.expression(1)); gen.visitJumpInsn(Opcodes.IFEQ, andFalse); typeStack.pop(); pushBoolean(true); gen.goTo(andEnd); gen.visitLabel(andFalse); pushBoolean(false); gen.visitLabel(andEnd); return null; } @Override public Void visitBoolor(JavascriptParser.BoolorContext ctx) { Label orTrue = new Label(); Label orEnd = new Label(); typeStack.push(Type.INT_TYPE); visit(ctx.expression(0)); gen.visitJumpInsn(Opcodes.IFNE, orTrue); visit(ctx.expression(1)); gen.visitJumpInsn(Opcodes.IFNE, orTrue); typeStack.pop(); pushBoolean(false); gen.goTo(orEnd); gen.visitLabel(orTrue); pushBoolean(true); gen.visitLabel(orEnd); return null; } @Override public Void visitConditional(JavascriptParser.ConditionalContext ctx) { Label condFalse = new Label(); Label condEnd = new Label(); typeStack.push(Type.INT_TYPE); visit(ctx.expression(0)); typeStack.pop(); gen.visitJumpInsn(Opcodes.IFEQ, condFalse); visit(ctx.expression(1)); gen.goTo(condEnd); gen.visitLabel(condFalse); visit(ctx.expression(2)); gen.visitLabel(condEnd); return null; } private void pushArith(int operator, ExpressionContext left, ExpressionContext right) { pushBinaryOp(operator, left, right, Type.DOUBLE_TYPE, Type.DOUBLE_TYPE, Type.DOUBLE_TYPE); } private void pushShift(int operator, ExpressionContext left, ExpressionContext right) { pushBinaryOp(operator, left, right, Type.LONG_TYPE, Type.INT_TYPE, Type.LONG_TYPE); } private void pushBitwise(int operator, ExpressionContext left, ExpressionContext right) { pushBinaryOp(operator, left, right, Type.LONG_TYPE, Type.LONG_TYPE, Type.LONG_TYPE); } private void pushBinaryOp(int operator, ExpressionContext left, ExpressionContext right, Type leftType, Type rightType, Type returnType) { typeStack.push(leftType); visit(left); typeStack.pop(); typeStack.push(rightType); visit(right); typeStack.pop(); gen.visitInsn(operator); gen.cast(returnType, typeStack.peek()); } private void pushCond(int operator, ExpressionContext left, ExpressionContext right) { Label labelTrue = new Label(); Label labelReturn = new Label(); typeStack.push(Type.DOUBLE_TYPE); visit(left); visit(right); typeStack.pop(); gen.ifCmp(Type.DOUBLE_TYPE, operator, labelTrue); pushBoolean(false); gen.goTo(labelReturn); gen.visitLabel(labelTrue); pushBoolean(true); gen.visitLabel(labelReturn); } private void pushBoolean(boolean truth) { switch (typeStack.peek().getSort()) { case Type.INT: gen.push(truth); break; case Type.LONG: gen.push(truth ? 1L : 0L); break; case Type.DOUBLE: gen.push(truth ? 1. : 0.); break; default: throw new IllegalStateException("Invalid expected type: " + typeStack.peek()); } } private void pushLong(long i) { switch (typeStack.peek().getSort()) { case Type.INT: gen.push((int) i); break; case Type.LONG: gen.push(i); break; case Type.DOUBLE: gen.push((double) i); break; default: throw new IllegalStateException("Invalid expected type: " + typeStack.peek()); } } }.visit(parseTree); gen.returnValue(); gen.endMethod(); classWriter.visitEnd(); } static String normalizeQuotes(String text) { StringBuilder out = new StringBuilder(text.length()); boolean inDoubleQuotes = false; for (int i = 0; i < text.length(); ++i) { char c = text.charAt(i); if (c == '\\') { c = text.charAt(++i); if (c == '\\') { out.append('\\'); // re-escape the backslash } // no escape for double quote } else if (c == '\'') { if (inDoubleQuotes) { // escape in output out.append('\\'); } else { int j = findSingleQuoteStringEnd(text, i); out.append(text, i, j); // copy up to end quote (leave end for append below) i = j; } } else if (c == '"') { c = '\''; // change beginning/ending doubles to singles inDoubleQuotes = !inDoubleQuotes; } out.append(c); } return out.toString(); } static int findSingleQuoteStringEnd(String text, int start) { ++start; // skip beginning while (text.charAt(start) != '\'') { if (text.charAt(start) == '\\') { ++start; // blindly consume escape value } ++start; } return start; } /** * The default set of functions available to expressions. * <p> * See the {@link org.apache.lucene.expressions.js package documentation} * for a list. */ public static final Map<String, Method> DEFAULT_FUNCTIONS; static { Map<String, Method> map = new HashMap<>(); try { final Properties props = new Properties(); try (Reader in = IOUtils.getDecodingReader(JavascriptCompiler.class, JavascriptCompiler.class.getSimpleName() + ".properties", StandardCharsets.UTF_8)) { props.load(in); } for (final String call : props.stringPropertyNames()) { final String[] vals = props.getProperty(call).split(","); if (vals.length != 3) { throw new Error("Syntax error while reading Javascript functions from resource"); } final Class<?> clazz = Class.forName(vals[0].trim()); final String methodName = vals[1].trim(); final int arity = Integer.parseInt(vals[2].trim()); @SuppressWarnings({ "rawtypes", "unchecked" }) Class[] args = new Class[arity]; Arrays.fill(args, double.class); Method method = clazz.getMethod(methodName, args); checkFunction(method); map.put(call, method); } } catch (ReflectiveOperationException | IOException e) { throw new Error("Cannot resolve function", e); } DEFAULT_FUNCTIONS = Collections.unmodifiableMap(map); } /** Check Method signature for compatibility. */ private static void checkFunction(Method method) { // check that the Method is public in some public reachable class: final MethodType type; try { type = MethodHandles.publicLookup().unreflect(method).type(); } catch (IllegalAccessException iae) { throw new IllegalArgumentException( method + " is not accessible (declaring class or method not public)."); } // do some checks if the signature is "compatible": if (!Modifier.isStatic(method.getModifiers())) { throw new IllegalArgumentException(method + " is not static."); } for (int arg = 0, arity = type.parameterCount(); arg < arity; arg++) { if (type.parameterType(arg) != double.class) { throw new IllegalArgumentException(method + " must take only double parameters."); } } if (type.returnType() != double.class) { throw new IllegalArgumentException(method + " does not return a double."); } } /** Cross check if declaring class of given method is the same as * returned by the given parent {@link ClassLoader} on string lookup. * This prevents {@link NoClassDefFoundError}. */ private static void checkFunctionClassLoader(Method method, ClassLoader parent) { boolean ok = false; try { final Class<?> clazz = method.getDeclaringClass(); ok = Class.forName(clazz.getName(), false, parent) == clazz; } catch (ClassNotFoundException e) { ok = false; } if (!ok) { throw new IllegalArgumentException( method + " is not declared by a class which is accessible by the given parent ClassLoader."); } } }