Skywalking-07:OAL原理——解释器实现
OAL
解释器实现
OAL
解释器是基于 Antlr4
实现的,我们先来了解下 Antlr4
Antlr4
基本介绍
Antlr4
使用案例
参考Antlr4的使用简介这篇文章,我们实现了一个简单的案例:antlr案例:简单的计算器,下面来讲讲这个案例。
首先,装好ANTLR v4(IDEA插件)插件,这个之后验证语法树的时候会用到。
在 pom.xml
中配置 antlr4
的依赖和插件
<dependency>
<groupId>org.antlr</groupId>
<artifactId>antlr4-runtime</artifactId>
<version>4.7.1</version>
</dependency>
<plugin>
<groupId>org.antlr</groupId>
<artifactId>antlr4-maven-plugin</artifactId>
<version>${antlr.version}</version>
<executions>
<execution>
<id>antlr</id>
<goals>
<goal>antlr4</goal>
</goals>
</execution>
</executions>
</plugin>
在 src/main/antlr4/com/switchvov/antlr/demo/calc
目录下添加一个 Calc.g4
文件
grammar Calc; //名称需要和文件名一致
root : expr EOF; //解决问题: no viable alternative at input "<EOF>"
expr
: expr "+" expr #add //标签会生成对应访问方法方便我们实现调用逻辑编写
| expr "-" expr #sub
| INT #int
;
INT : [0-9]+ //定义整数
;
WS : [
]+ -> skip //跳过空白类字符
;
执行一下: mvn compile -Dmaven.test.skip=true
,在 target/generated-sources/antlr4
会生成相应的 Java
代码。
使用方式默认是监听器模式,也可以配置成访问者模式。
监听器模式:主要借助了 ParseTreeWalker
这样一个类,相当于是一个 hook
,每经过一个树的节点,便会触发对应节点的方法。好处就算是比较方便,但是灵活性不够,不能够自主性的调用任意节点进行使用。
访问者模式:将每个数据的节点类型高度抽象出来够,根据你传入的上下文类型来判断你想要访问的是哪个节点,触发对应的方法
PS:结论,简单语法监听器模式就可以了,如果语法比较灵活可以考虑使用访问者模式。
antlr4
├── Calc.tokens
├── CalcLexer.tokens
└── com
└── switchvov
└── antlr
└── demo
└── calc
├── Calc.interp
├── CalcBaseListener.java # 监听模式下生成的监听器基类,实现类监听器接口,通过继承该类可以实现相应的功能
├── CalcLexer.interp
├── CalcLexer.java # 词法解析器
├── CalcListener.java # 监听模式下生成的监听器接口
└── CalcParser.java # 语法解析器
继承 com.switchvov.antlr.demo.calc.CalcBaseListener
,实现计算器相应功能
package com.switchvov.antlr.demo.calc;
import java.util.ArrayDeque;
import java.util.Deque;
/**
* @author switch
* @since 2021/6/30
*/
public class CalcExecuteListener extends CalcBaseListener {
Deque<Integer> queue = new ArrayDeque<>(16);
@Override
public void exitInt(CalcParser.IntContext ctx) {
queue.add(Integer.parseInt(ctx.INT().getText()));
}
@Override
public void exitAdd(CalcParser.AddContext ctx) {
int r = queue.pop();
int l = queue.pop();
queue.add(l + r);
}
@Override
public void exitSub(CalcParser.SubContext ctx) {
int r = queue.pop();
int l = queue.pop();
queue.add(l - r);
}
public int result() {
return queue.pop();
}
}
测试一下
package com.switchvov.antlr.demo.calc;
import org.antlr.v4.runtime.CharStreams;
import org.antlr.v4.runtime.CodePointCharStream;
import org.antlr.v4.runtime.CommonTokenStream;
import org.antlr.v4.runtime.tree.ParseTree;
import org.antlr.v4.runtime.tree.ParseTreeWalker;
import org.junit.Test;
/**
* @author switch
* @since 2021/6/30
*/
public class CalcTest {
public static int exec(String input) {
// 读入字符串
CodePointCharStream cs = CharStreams.fromString(input);
// 词法解析
CalcLexer lexer = new CalcLexer(cs);
CommonTokenStream tokens = new CommonTokenStream(lexer);
// 语法解析
CalcParser parser = new CalcParser(tokens);
// 监听器触发获取执行结果
ParseTree tree = parser.expr();
ParseTreeWalker walker = new ParseTreeWalker();
CalcExecuteListener listener = new CalcExecuteListener();
walker.walk(listener, tree);
return listener.result();
}
@Test
public void testCalc() {
String input = "1+2";
// 输出结果:3
System.out.println(exec(input));
}
}
Antlr4 IDEA
插件使用
在 Calc.g4
语法定义文件中,鼠标右击可以选择 Test Rule root
,然后在 ANTLR Preview
的输入框中填入 1 + 2
就可以校验语法文件是否 OK
,并且也可以看到相应的语法树
Antlr4
在 Skywalking
的应用
通过“ Antlr4
基本介绍”一节,基本上对 Antlr4
使用有了个大概的认识。下面来看看 Skywalking
中 Antlr4
是如何使用的。
词法定义
在 oap-server/oal-grammar/src/main/antlr4/org/apache/skywalking/oal/rt/grammar/OALLexer.g4
文件中,我们能看到 OAL
的词法定义
// Observability Analysis Language lexer
lexer grammar OALLexer;
@Header {package org.apache.skywalking.oal.rt.grammar;}
// Keywords
FROM: "from";
FILTER: "filter";
DISABLE: "disable";
SRC_ALL: "All";
SRC_SERVICE: "Service";
SRC_SERVICE_INSTANCE: "ServiceInstance";
SRC_ENDPOINT: "Endpoint";
SRC_SERVICE_RELATION: "ServiceRelation";
SRC_SERVICE_INSTANCE_RELATION: "ServiceInstanceRelation";
SRC_ENDPOINT_RELATION: "EndpointRelation";
SRC_SERVICE_INSTANCE_JVM_CPU: "ServiceInstanceJVMCPU";
SRC_SERVICE_INSTANCE_JVM_MEMORY: "ServiceInstanceJVMMemory";
SRC_SERVICE_INSTANCE_JVM_MEMORY_POOL: "ServiceInstanceJVMMemoryPool";
SRC_SERVICE_INSTANCE_JVM_GC: "ServiceInstanceJVMGC";
SRC_SERVICE_INSTANCE_JVM_THREAD: "ServiceInstanceJVMThread";
SRC_SERVICE_INSTANCE_JVM_CLASS:"ServiceInstanceJVMClass";
SRC_DATABASE_ACCESS: "DatabaseAccess";
SRC_SERVICE_INSTANCE_CLR_CPU: "ServiceInstanceCLRCPU";
SRC_SERVICE_INSTANCE_CLR_GC: "ServiceInstanceCLRGC";
SRC_SERVICE_INSTANCE_CLR_THREAD: "ServiceInstanceCLRThread";
SRC_ENVOY_INSTANCE_METRIC: "EnvoyInstanceMetric";
// Browser keywords
SRC_BROWSER_APP_PERF: "BrowserAppPerf";
SRC_BROWSER_APP_PAGE_PERF: "BrowserAppPagePerf";
SRC_BROWSER_APP_SINGLE_VERSION_PERF: "BrowserAppSingleVersionPerf";
SRC_BROWSER_APP_TRAFFIC: "BrowserAppTraffic";
SRC_BROWSER_APP_PAGE_TRAFFIC: "BrowserAppPageTraffic";
SRC_BROWSER_APP_SINGLE_VERSION_TRAFFIC: "BrowserAppSingleVersionTraffic";
// Constructors symbols
DOT: ".";
LR_BRACKET: "(";
RR_BRACKET: ")";
LS_BRACKET: "[";
RS_BRACKET: "]";
COMMA: ",";
SEMI: ";";
EQUAL: "=";
DUALEQUALS: "==";
ALL: "*";
GREATER: ">";
LESS: "<";
GREATER_EQUAL: ">=";
LESS_EQUAL: "<=";
NOT_EQUAL: "!=";
LIKE: "like";
IN: "in";
CONTAIN: "contain";
NOT_CONTAIN: "not contain";
// Literals
BOOL_LITERAL: "true"
| "false"
;
NUMBER_LITERAL : Digits+;
CHAR_LITERAL: """ (~["\
] | EscapeSequence) """;
STRING_LITERAL: """ (~["\
] | EscapeSequence)* """;
DelimitedComment
: "/*" ( DelimitedComment | . )*? "*/"
-> channel(HIDDEN)
;
LineComment
: "//" ~[u000Au000D]*
-> channel(HIDDEN)
;
SPACE: [
]+ -> channel(HIDDEN);
// Identifiers
IDENTIFIER: Letter LetterOrDigit*;
// Fragment rules
fragment EscapeSequence
: "\" [btnfr""\]
| "\" ([0-3]? [0-7])? [0-7]
| "\" "u"+ HexDigit HexDigit HexDigit HexDigit
;
fragment HexDigits
: HexDigit ((HexDigit | "_")* HexDigit)?
;
fragment HexDigit
: [0-9a-fA-F]
;
fragment Digits
: [0-9] ([0-9_]* [0-9])?
;
fragment LetterOrDigit
: Letter
| [0-9]
;
fragment Letter
: [a-zA-Z$_] // these are the "java letters" below 0x7F
| ~[u0000-u007FuD800-uDBFF] // covers all characters above 0x7F which are not a surrogate
| [uD800-uDBFF] [uDC00-uDFFF] // covers UTF-16 surrogate pairs encodings for U+10000 to U+10FFFF
;
语法定义
在 oap-server/oal-grammar/src/main/antlr4/org/apache/skywalking/oal/rt/grammar/OALParser.g4
文件中,我们能看到 OAL
的语法定义
parser grammar OALParser;
@Header {package org.apache.skywalking.oal.rt.grammar;}
options { tokenVocab=OALLexer; }
// Top Level Description
root
: (aggregationStatement | disableStatement)*
;
aggregationStatement
: variable (SPACE)? EQUAL (SPACE)? metricStatement DelimitedComment? LineComment? (SEMI|EOF)
;
disableStatement
: DISABLE LR_BRACKET disableSource RR_BRACKET DelimitedComment? LineComment? (SEMI|EOF)
;
metricStatement
: FROM LR_BRACKET source (sourceAttributeStmt+) RR_BRACKET (filterStatement+)? DOT aggregateFunction
;
filterStatement
: DOT FILTER LR_BRACKET filterExpression RR_BRACKET
;
filterExpression
: expression
;
source
: SRC_ALL | SRC_SERVICE | SRC_DATABASE_ACCESS | SRC_SERVICE_INSTANCE | SRC_ENDPOINT |
SRC_SERVICE_RELATION | SRC_SERVICE_INSTANCE_RELATION | SRC_ENDPOINT_RELATION |
SRC_SERVICE_INSTANCE_JVM_CPU | SRC_SERVICE_INSTANCE_JVM_MEMORY | SRC_SERVICE_INSTANCE_JVM_MEMORY_POOL | SRC_SERVICE_INSTANCE_JVM_GC | SRC_SERVICE_INSTANCE_JVM_THREAD | SRC_SERVICE_INSTANCE_JVM_CLASS |// JVM source of service instance
SRC_SERVICE_INSTANCE_CLR_CPU | SRC_SERVICE_INSTANCE_CLR_GC | SRC_SERVICE_INSTANCE_CLR_THREAD |
SRC_ENVOY_INSTANCE_METRIC |
SRC_BROWSER_APP_PERF | SRC_BROWSER_APP_PAGE_PERF | SRC_BROWSER_APP_SINGLE_VERSION_PERF |
SRC_BROWSER_APP_TRAFFIC | SRC_BROWSER_APP_PAGE_TRAFFIC | SRC_BROWSER_APP_SINGLE_VERSION_TRAFFIC
;
disableSource
: IDENTIFIER
;
sourceAttributeStmt
: DOT sourceAttribute
;
sourceAttribute
: IDENTIFIER | ALL
;
variable
: IDENTIFIER
;
aggregateFunction
: functionName LR_BRACKET ((funcParamExpression (COMMA funcParamExpression)?) | (literalExpression (COMMA literalExpression)?))? RR_BRACKET
;
functionName
: IDENTIFIER
;
funcParamExpression
: expression
;
literalExpression
: BOOL_LITERAL | NUMBER_LITERAL | IDENTIFIER
;
expression
: booleanMatch | stringMatch | greaterMatch | lessMatch | greaterEqualMatch | lessEqualMatch | notEqualMatch | booleanNotEqualMatch | likeMatch | inMatch | containMatch | notContainMatch
;
containMatch
: conditionAttributeStmt CONTAIN stringConditionValue
;
notContainMatch
: conditionAttributeStmt NOT_CONTAIN stringConditionValue
;
booleanMatch
: conditionAttributeStmt DUALEQUALS booleanConditionValue
;
stringMatch
: conditionAttributeStmt DUALEQUALS (stringConditionValue | enumConditionValue)
;
greaterMatch
: conditionAttributeStmt GREATER numberConditionValue
;
lessMatch
: conditionAttributeStmt LESS numberConditionValue
;
greaterEqualMatch
: conditionAttributeStmt GREATER_EQUAL numberConditionValue
;
lessEqualMatch
: conditionAttributeStmt LESS_EQUAL numberConditionValue
;
booleanNotEqualMatch
: conditionAttributeStmt NOT_EQUAL booleanConditionValue
;
notEqualMatch
: conditionAttributeStmt NOT_EQUAL (numberConditionValue | stringConditionValue | enumConditionValue)
;
likeMatch
: conditionAttributeStmt LIKE stringConditionValue
;
inMatch
: conditionAttributeStmt IN multiConditionValue
;
multiConditionValue
: LS_BRACKET (numberConditionValue ((COMMA numberConditionValue)*) | stringConditionValue ((COMMA stringConditionValue)*) | enumConditionValue ((COMMA enumConditionValue)*)) RS_BRACKET
;
conditionAttributeStmt
: conditionAttribute ((DOT conditionAttribute)*)
;
conditionAttribute
: IDENTIFIER
;
booleanConditionValue
: BOOL_LITERAL
;
stringConditionValue
: STRING_LITERAL
;
enumConditionValue
: IDENTIFIER DOT IDENTIFIER
;
numberConditionValue
: NUMBER_LITERAL
;
Antlr4
生成 Java
代码
在 oap-server/oal-grammar
下执行 mvn compile -Dmaven.test.skip=true
会在 oap-server/oal-grammar/target/generated-sources/antlr4
目录下生成相应的 Java
代码
.
├── OALLexer.tokens
├── OALParser.tokens
└── org
└── apache
└── skywalking
└── oal
└── rt
└── grammar
├── OALLexer.interp
├── OALLexer.java # 词法解析器
├── OALParser.interp
├── OALParser.java # 语法解析器
├── OALParserBaseListener.java # 监听器
└── OALParserListener.java
在 Skywalking
的使用
通过“ Antlr4
使用案例”一节,可以知道 Antlr4
有两种功能实现方式:监听器或者访问器。
通过“ Antlr4
生成 Java
代码”一节,知道 Skywalking
使用的是监听器模式。
Skywalking
关于 OAL
的相应的代码都在 oap-server/oal-rt
模块中。
org.apache.skywalking.oal.rt.grammar.OALParserBaseListener
的继承类坐标是 org.apache.skywalking.oal.rt.parser.OALListener
package org.apache.skywalking.oal.rt.parser;
import java.util.Arrays;
import java.util.List;
import org.antlr.v4.runtime.misc.NotNull;
import org.apache.skywalking.oal.rt.grammar.OALParser;
import org.apache.skywalking.oal.rt.grammar.OALParserBaseListener;
import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine;
public class OALListener extends OALParserBaseListener {
private List<AnalysisResult> results;
private AnalysisResult current;
private DisableCollection collection;
private ConditionExpression conditionExpression;
private final String sourcePackage;
public OALListener(OALScripts scripts, String sourcePackage) {
this.results = scripts.getMetricsStmts();
this.collection = scripts.getDisableCollection();
this.sourcePackage = sourcePackage;
}
@Override
public void enterAggregationStatement(@NotNull OALParser.AggregationStatementContext ctx) {
current = new AnalysisResult();
}
@Override
public void exitAggregationStatement(@NotNull OALParser.AggregationStatementContext ctx) {
DeepAnalysis deepAnalysis = new DeepAnalysis();
results.add(deepAnalysis.analysis(current));
current = null;
}
@Override
public void enterSource(OALParser.SourceContext ctx) {
current.setSourceName(ctx.getText());
current.setSourceScopeId(DefaultScopeDefine.valueOf(metricsNameFormat(ctx.getText())));
}
@Override
public void enterSourceAttribute(OALParser.SourceAttributeContext ctx) {
current.getSourceAttribute().add(ctx.getText());
}
@Override
public void enterVariable(OALParser.VariableContext ctx) {
}
@Override
public void exitVariable(OALParser.VariableContext ctx) {
current.setVarName(ctx.getText());
current.setMetricsName(metricsNameFormat(ctx.getText()));
current.setTableName(ctx.getText().toLowerCase());
}
@Override
public void enterFunctionName(OALParser.FunctionNameContext ctx) {
current.setAggregationFunctionName(ctx.getText());
}
@Override
public void enterFilterStatement(OALParser.FilterStatementContext ctx) {
conditionExpression = new ConditionExpression();
}
@Override
public void exitFilterStatement(OALParser.FilterStatementContext ctx) {
current.addFilterExpressionsParserResult(conditionExpression);
conditionExpression = null;
}
@Override
public void enterFuncParamExpression(OALParser.FuncParamExpressionContext ctx) {
conditionExpression = new ConditionExpression();
}
@Override
public void exitFuncParamExpression(OALParser.FuncParamExpressionContext ctx) {
current.addFuncConditionExpression(conditionExpression);
conditionExpression = null;
}
/////////////
// Expression
////////////
@Override
public void enterConditionAttribute(OALParser.ConditionAttributeContext ctx) {
conditionExpression.getAttributes().add(ctx.getText());
}
@Override
public void enterBooleanMatch(OALParser.BooleanMatchContext ctx) {
conditionExpression.setExpressionType("booleanMatch");
}
@Override
public void enterStringMatch(OALParser.StringMatchContext ctx) {
conditionExpression.setExpressionType("stringMatch");
}
@Override
public void enterGreaterMatch(OALParser.GreaterMatchContext ctx) {
conditionExpression.setExpressionType("greaterMatch");
}
@Override
public void enterGreaterEqualMatch(OALParser.GreaterEqualMatchContext ctx) {
conditionExpression.setExpressionType("greaterEqualMatch");
}
@Override
public void enterLessMatch(OALParser.LessMatchContext ctx) {
conditionExpression.setExpressionType("lessMatch");
}
@Override
public void enterLessEqualMatch(OALParser.LessEqualMatchContext ctx) {
conditionExpression.setExpressionType("lessEqualMatch");
}
@Override
public void enterNotEqualMatch(final OALParser.NotEqualMatchContext ctx) {
conditionExpression.setExpressionType("notEqualMatch");
}
@Override
public void enterBooleanNotEqualMatch(final OALParser.BooleanNotEqualMatchContext ctx) {
conditionExpression.setExpressionType("booleanNotEqualMatch");
}
@Override
public void enterLikeMatch(final OALParser.LikeMatchContext ctx) {
conditionExpression.setExpressionType("likeMatch");
}
@Override
public void enterContainMatch(final OALParser.ContainMatchContext ctx) {
conditionExpression.setExpressionType("containMatch");
}
@Override
public void enterNotContainMatch(final OALParser.NotContainMatchContext ctx) {
conditionExpression.setExpressionType("notContainMatch");
}
@Override
public void enterInMatch(final OALParser.InMatchContext ctx) {
conditionExpression.setExpressionType("inMatch");
}
@Override
public void enterMultiConditionValue(final OALParser.MultiConditionValueContext ctx) {
conditionExpression.enterMultiConditionValue();
}
@Override
public void exitMultiConditionValue(final OALParser.MultiConditionValueContext ctx) {
conditionExpression.exitMultiConditionValue();
}
@Override
public void enterBooleanConditionValue(OALParser.BooleanConditionValueContext ctx) {
enterConditionValue(ctx.getText());
}
@Override
public void enterStringConditionValue(OALParser.StringConditionValueContext ctx) {
enterConditionValue(ctx.getText());
}
@Override
public void enterEnumConditionValue(OALParser.EnumConditionValueContext ctx) {
enterConditionValue(ctx.getText());
}
@Override
public void enterNumberConditionValue(OALParser.NumberConditionValueContext ctx) {
conditionExpression.isNumber();
enterConditionValue(ctx.getText());
}
private void enterConditionValue(String value) {
if (value.split("\.").length == 2 && !value.startsWith(""")) {
// Value is an enum.
value = sourcePackage + value;
}
conditionExpression.addValue(value);
}
/////////////
// Expression end.
////////////
@Override
public void enterLiteralExpression(OALParser.LiteralExpressionContext ctx) {
if (ctx.IDENTIFIER() == null) {
current.addFuncArg(new Argument(EntryMethod.LITERAL_TYPE, Arrays.asList(ctx.getText())));
return;
}
current.addFuncArg(new Argument(EntryMethod.IDENTIFIER_TYPE, Arrays.asList(ctx.getText().split("\."))));
}
private String metricsNameFormat(String source) {
source = firstLetterUpper(source);
int idx;
while ((idx = source.indexOf("_")) > -1) {
source = source.substring(0, idx) + firstLetterUpper(source.substring(idx + 1));
}
return source;
}
/**
* Disable source
*/
@Override
public void enterDisableSource(OALParser.DisableSourceContext ctx) {
collection.add(ctx.getText());
}
private String firstLetterUpper(String source) {
return source.substring(0, 1).toUpperCase() + source.substring(1);
}
}
简单来说,就是通过监听器封装了个 org.apache.skywalking.oal.rt.parser.OALScripts
对象
package org.apache.skywalking.oal.rt.parser;
import java.util.LinkedList;
import java.util.List;
import lombok.Getter;
@Getter
public class OALScripts {
// 解析出来的分析结果集合
private List<AnalysisResult> metricsStmts;
// 禁用表达式集合
private DisableCollection disableCollection;
public OALScripts() {
metricsStmts = new LinkedList<>();
disableCollection = new DisableCollection();
}
}
org.apache.skywalking.oal.rt.parser.ScriptParser
类读取 oal
文件,使用 Antlr
生成的 Java
类进行解析
package org.apache.skywalking.oal.rt.parser;
import java.io.IOException;
import java.io.Reader;
import org.antlr.v4.runtime.CharStreams;
import org.antlr.v4.runtime.CommonTokenStream;
import org.antlr.v4.runtime.tree.ParseTree;
import org.antlr.v4.runtime.tree.ParseTreeWalker;
import org.apache.skywalking.oal.rt.grammar.OALLexer;
import org.apache.skywalking.oal.rt.grammar.OALParser;
/**
* Script reader and parser.
*/
public class ScriptParser {
private OALLexer lexer;
private String sourcePackage;
private ScriptParser() {
}
public static ScriptParser createFromFile(Reader scriptReader, String sourcePackage) throws IOException {
ScriptParser parser = new ScriptParser();
parser.lexer = new OALLexer(CharStreams.fromReader(scriptReader));
parser.sourcePackage = sourcePackage;
return parser;
}
public static ScriptParser createFromScriptText(String script, String sourcePackage) throws IOException {
ScriptParser parser = new ScriptParser();
parser.lexer = new OALLexer(CharStreams.fromString(script));
parser.sourcePackage = sourcePackage;
return parser;
}
public OALScripts parse() throws IOException {
OALScripts scripts = new OALScripts();
CommonTokenStream tokens = new CommonTokenStream(lexer);
OALParser parser = new OALParser(tokens);
ParseTree tree = parser.root();
ParseTreeWalker walker = new ParseTreeWalker();
walker.walk(new OALListener(scripts, sourcePackage), tree);
return scripts;
}
public void close() {
}
}
参考文档
- ANTLR官网
- antlr4 GitHub
- antlr4 语法案例
- Antlr4的使用简介
- antlr案例:简单的计算器
- 某小伙的Antlr4学习笔记
- ANTLR v4(IDEA插件)
分享并记录所学所见