From f384f4d36bb19c49c48f469e979ecaa84e885f60 Mon Sep 17 00:00:00 2001 From: hongqiaowei Date: Thu, 10 Jun 2021 10:19:49 +0800 Subject: [PATCH 01/25] Publish v2.1.0-beta1 --- README.en-us.md | 3 ++- README.md | 3 ++- fizz-bootstrap/pom.xml | 2 +- fizz-common/pom.xml | 2 +- fizz-core/pom.xml | 2 +- fizz-plugin/pom.xml | 2 +- fizz-spring-boot-starter/pom.xml | 2 +- pom.xml | 2 +- 8 files changed, 10 insertions(+), 8 deletions(-) diff --git a/README.en-us.md b/README.en-us.md index c38ef3e..19ba0cc 100644 --- a/README.en-us.md +++ b/README.en-us.md @@ -4,7 +4,7 @@ English | [简体中文](./README.md)

- Version + Version Documentation @@ -99,6 +99,7 @@ Starting from v1.3.0, the frontend and backend of the management backend are mer | v1.5.0 | v1.5.0 | | v1.5.1 | v1.5.1 | | v2.0.0 | v2.0.0 | +| v2.1.0 | v2.1.0 | Please download the corresponding management backend version according to the version of the community version diff --git a/README.md b/README.md index cd03dad..0daea21 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@

- Version + Version Documentation @@ -98,6 +98,7 @@ API地址：http://demo.fizzgate.com/proxy/[服务名]/[API_Path] | v1.5.0 | v1.5.0 | | v1.5.1 | v1.5.1 | | v2.0.0 | v2.0.0 | +| v2.1.0 | v2.1.0 | 请根据社区版的版本下载对应的管理后台版本 diff --git a/fizz-bootstrap/pom.xml b/fizz-bootstrap/pom.xml index 11a701c..9354ffe 100644 --- a/fizz-bootstrap/pom.xml +++ b/fizz-bootstrap/pom.xml @@ -12,7 +12,7 @@ com.fizzgate fizz-bootstrap - 2.0.0 + 2.1.0-beta1 1.8 diff --git a/fizz-common/pom.xml b/fizz-common/pom.xml index 30e5d37..a413b99 100644 --- a/fizz-common/pom.xml +++ b/fizz-common/pom.xml @@ -5,7 +5,7 @@ fizz-gateway-community com.fizzgate - 2.0.0 + 2.1.0-beta1 ../pom.xml 4.0.0 diff --git a/fizz-core/pom.xml b/fizz-core/pom.xml index b220edf..6813385 100644 --- a/fizz-core/pom.xml +++ b/fizz-core/pom.xml @@ -5,7 +5,7 @@ fizz-gateway-community com.fizzgate - 2.0.0 + 2.1.0-beta1 ../pom.xml 4.0.0 diff --git a/fizz-plugin/pom.xml b/fizz-plugin/pom.xml index 648af6f..92e3cfd 100644 --- a/fizz-plugin/pom.xml +++ b/fizz-plugin/pom.xml @@ -5,7 +5,7 @@ fizz-gateway-community com.fizzgate - 2.0.0 + 2.1.0-beta1 ../pom.xml 4.0.0 diff --git a/fizz-spring-boot-starter/pom.xml b/fizz-spring-boot-starter/pom.xml index a35041b..822c5c0 100644 --- a/fizz-spring-boot-starter/pom.xml +++ b/fizz-spring-boot-starter/pom.xml @@ -5,7 +5,7 @@ fizz-gateway-community com.fizzgate - 2.0.0 + 2.1.0-beta1 ../pom.xml 4.0.0 diff --git a/pom.xml b/pom.xml index 64f5ff5..31953da 100644 --- a/pom.xml +++ b/pom.xml @@ -30,7 +30,7 @@ fizz-gateway-community ${project.artifactId} fizz gateway community - 2.0.0 + 2.1.0-beta1 pom fizz-common From 5fc2343db07dfc4ccdf540b2ecd23be9a70468bc Mon Sep 17 00:00:00 2001 From: hongqiaowei Date: Thu, 10 Jun 2021 10:32:48 +0800 Subject: [PATCH 02/25] Release v2.1.0 --- fizz-bootstrap/pom.xml | 2 +- fizz-common/pom.xml | 2 +- fizz-core/pom.xml | 2 +- fizz-plugin/pom.xml | 2 +- fizz-spring-boot-starter/pom.xml | 2 +- pom.xml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/fizz-bootstrap/pom.xml b/fizz-bootstrap/pom.xml index 9354ffe..5d39cf8 100644 --- a/fizz-bootstrap/pom.xml +++ b/fizz-bootstrap/pom.xml @@ -12,7 +12,7 @@ com.fizzgate fizz-bootstrap - 2.1.0-beta1 + 2.1.0 1.8 diff --git a/fizz-common/pom.xml b/fizz-common/pom.xml index a413b99..53fde5e 100644 --- a/fizz-common/pom.xml +++ b/fizz-common/pom.xml @@ -5,7 +5,7 @@ fizz-gateway-community com.fizzgate - 2.1.0-beta1 + 2.1.0 ../pom.xml 4.0.0 diff --git a/fizz-core/pom.xml b/fizz-core/pom.xml index 6813385..5e4efa7 100644 --- a/fizz-core/pom.xml +++ b/fizz-core/pom.xml @@ -5,7 +5,7 @@ fizz-gateway-community com.fizzgate - 2.1.0-beta1 + 2.1.0 ../pom.xml 4.0.0 diff --git a/fizz-plugin/pom.xml b/fizz-plugin/pom.xml index 92e3cfd..1022bba 100644 --- a/fizz-plugin/pom.xml +++ b/fizz-plugin/pom.xml @@ -5,7 +5,7 @@ fizz-gateway-community com.fizzgate - 2.1.0-beta1 + 2.1.0 ../pom.xml 4.0.0 diff --git a/fizz-spring-boot-starter/pom.xml b/fizz-spring-boot-starter/pom.xml index 822c5c0..6f4f739 100644 --- a/fizz-spring-boot-starter/pom.xml +++ b/fizz-spring-boot-starter/pom.xml @@ -5,7 +5,7 @@ fizz-gateway-community com.fizzgate - 2.1.0-beta1 + 2.1.0 ../pom.xml 4.0.0 diff --git a/pom.xml b/pom.xml index 31953da..a008476 100644 --- a/pom.xml +++ b/pom.xml @@ -30,7 +30,7 @@ fizz-gateway-community ${project.artifactId} fizz gateway community - 2.1.0-beta1 + 2.1.0 pom fizz-common From e5bae079af4605648b219f93dda36eaa4ddd2935 Mon Sep 17 00:00:00 2001 From: linwaiwai Date: Fri, 18 Jun 2021 13:31:17 +0800 Subject: [PATCH 03/25] update README with scene pic --- README.en-us.md | 16 ++++++++++++---- README.md | 17 +++++++++++++---- 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/README.en-us.md b/README.en-us.md index 19ba0cc..c03167d 100644 --- a/README.en-us.md +++ b/README.en-us.md @@ -20,7 +20,7 @@ English | [简体中文](./README.md) ## What 's Fizz Gateway？ -A Managerment API Gateway in Java. Fizz Gateway is a Java-based microservice gateway that can realize hot service aggregation, automatic authorization selection, online service script coding, online testing, high-performance routing, API audit management and other purposes. It has a powerful The custom plug-in system can be extended by youself, and provides a friendly graphical configuration interface, which can quickly help enterprises to manage API services, reduce middle layer glue code, reduce coding investment, and improve the stability and security of API services. +An Aggregation API Gateway in Java . Fizz Gateway is a Java-based microservice gateway that can realize hot service aggregation, automatic authorization selection, online service script coding, online testing, high-performance routing, API audit management and other purposes. It has a powerful The custom plug-in system can be extended by youself, and provides a friendly graphical configuration interface, which can quickly help enterprises to manage API services, reduce middle layer glue code, reduce coding investment, and improve the stability and security of API services. ## Demo @@ -37,6 +37,10 @@ API access：http://demo.fizzgate.com/proxy/[Service Name]/[API Path] +## Fizz's typical scene + + + ## Product Features - Cluster management: Fizz gateway nodes are stateless with configuration information that is automatically synchronized, and horizontal expansion of nodes and multi-cluster deployment are supported. @@ -122,7 +126,7 @@ Dependent installation can refer to detailed deployment tutorial #### 一、Install management backend -Download the fizz-manager-professional installation package from github's releases (https://github.com/wehotel/fizz-gateway-community/releases) +Download the fizz-manager-professional installation package from github's releases (https://wj.qq.com/s2/8682608/8fe2/) ##### Management backend (fizz-manager-professional) @@ -218,7 +222,11 @@ Fizz官方技术交流②群（已满） Fizz官方技术交流③群：512164278 -![](https://user-images.githubusercontent.com/184315/97130743-3572d100-177d-11eb-97c8-7599a22c7c04.png) + + +Fizz offical wechat group + + ## Related acticles @@ -238,7 +246,7 @@ Fizz官方技术交流③群：512164278 1. The fizz-gateway-community community version of the gateway core project is open in GNU v3 and can be used for free. -2. Management backend projects (fizz-manager-professional) as commercial versions only open binary packages [free download](https://github.com/wehotel/fizz-gateway-community/releases), and For commercial projects, please contact us (sale@fizzgate.com) for authorization. +2. Management backend projects (fizz-manager-professional) as commercial versions only open binary packages [free download](https://wj.qq.com/s2/8682608/8fe2/), and For commercial projects, please contact us (sale@fizzgate.com) for authorization. ## System screenshot diff --git a/README.md b/README.md index 0daea21..9fbf4d9 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,10 @@ API地址：http://demo.fizzgate.com/proxy/[服务名]/[API_Path] +## Fizz典型应用场景 + + + ## 产品特性 - 集群管理：Fizz网关节点是无状态的，配置信息自动同步，支持节点水平拓展和多集群部署。 @@ -51,7 +55,8 @@ API地址：http://demo.fizzgate.com/proxy/[服务名]/[API_Path] - 高性能：性能在众多网关之中表现优异。 - 版本控制：支持操作的发布和多次回滚。 - 管理后台：通过管理后台界面对网关集群进行各项配置。 -- 回调管理：支持回调的管理、订阅、重放、以及日志 +- 回调管理：支持回调的管理、订阅、重放、以及日志。 +- 多级限流：细颗粒度的限流方式包含服务限流，接口限流，APP_ID限流，IP限流 ## 基准测试 @@ -121,7 +126,7 @@ API地址：http://demo.fizzgate.com/proxy/[服务名]/[API_Path] #### 一、安装管理后台 -从github的releases(https://github.com/wehotel/fizz-gateway-community/releases) 下载 fizz-manager-professional 安装包 +从github的releases(https://wj.qq.com/s2/8682608/8fe2/) 下载 fizz-manager-professional 安装包 ##### 管理后台（fizz-manager-professional） @@ -215,7 +220,11 @@ Fizz官方技术交流②群（已满） Fizz官方技术交流③群：512164278 -![](https://user-images.githubusercontent.com/184315/97130743-3572d100-177d-11eb-97c8-7599a22c7c04.png) + + +Fizz官方微信群 + + ## 相关文章 @@ -235,7 +244,7 @@ Fizz官方技术交流③群：512164278 1. 网关核心项目fizz-gateway-community社区版本以GNU v3的方式进行的开放，可以免费使用。 -2. 管理后台项目(fizz-manager-professional)作为商业版本仅开放二进制包 [免费下载](https://github.com/wehotel/fizz-gateway-community/releases)，而商业项目请联系我们（sale@fizzgate.com）进行授权。 +2. 管理后台项目(fizz-manager-professional)作为商业版本仅开放二进制包 [免费下载](https://wj.qq.com/s2/8682608/8fe2/)，而商业项目请联系我们（sale@fizzgate.com）进行授权。 ## 系统截图 From 89050ab22968ce9078a9027f5c81df568f04e492 Mon Sep 17 00:00:00 2001 From: zhongjie Date: Tue, 15 Jun 2021 11:26:48 +0800 Subject: [PATCH 04/25] add Win boot script --- .github/workflows/maven.yml | 2 + README.en-us.md | 3 +- README.md | 3 +- fizz-bootstrap/pom.xml | 2 + fizz-bootstrap/sh/boot.cmd | 69 +++++++++++++++++++ .../src/main/resources/application.yml | 2 +- 6 files changed, 78 insertions(+), 3 deletions(-) create mode 100644 fizz-bootstrap/sh/boot.cmd diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index a505dfe..1862d18 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -51,8 +51,10 @@ jobs: mkdir fizz-gateway-community cp ./fizz-bootstrap/target/fizz-bootstrap-*.jar ./fizz-gateway-community/fizz-gateway-community.jar cp ./fizz-bootstrap/target/classes/application.yml ./fizz-gateway-community/application.yml + cp ./fizz-bootstrap/target/classes/bootstrap.yml ./fizz-gateway-community/bootstrap.yml cp ./fizz-bootstrap/target/classes/log4j2-spring.xml ./fizz-gateway-community/log4j2-spring.xml cp ./fizz-bootstrap/sh/boot.sh ./fizz-gateway-community/boot.sh + cp ./fizz-bootstrap/sh/boot.cmd ./fizz-gateway-community/boot.cmd zip -r ./fizz-gateway-community-${{ steps.releaseVersion.outputs.substring }}.zip ./fizz-gateway-community/* - name: Release uses: softprops/action-gh-release@v1 diff --git a/README.en-us.md b/README.en-us.md index c03167d..72f2323 100644 --- a/README.en-us.md +++ b/README.en-us.md @@ -181,7 +181,8 @@ Installation method 1: binary package: 1. Download the latest binary package of fizz-gateway-community and upzip to a directory, modify the configuration of the configuration center, registry, and redis in the application.yml configuration file (redis configuration needs to be consistent with the management backend). 2. Modify the apollo connection and JVM memory configuration of the boot.sh script -3. Execute `./boot.sh start` command to start the service, support start/stop/restart/status command +3. Linux startup Execute `./boot.sh start` command to start the service, support start/stop/restart/status command +4. Windows startup Execute `.\boot.cmd start` command to start the service, support start/stop/restart/status command Installation method 2: source code: diff --git a/README.md b/README.md index 9fbf4d9..fd2835e 100644 --- a/README.md +++ b/README.md @@ -180,7 +180,8 @@ docker run --rm -d -p 8000:8000 \ 1. 下载fizz-gateway-community的二进制安装包，解压修改application.yml配置文件里配置中心、注册中心、redis(redis配置需与管理后台一致)的配置 2. 根据需要修改boot.sh脚本的apollo连接，不使用apollo配置中心可跳过 -3. 执行 `./boot.sh start` 命令启动服务，支持 start/stop/restart/status命令 +3. Linux启动 执行 `./boot.sh start` 命令启动服务，支持 start/stop/restart/status命令 +4. Windows启动 执行`.\boot.cmd start` 命令启动服务，支持 start/stop/restart/status命令 安装方式二：源码安装: diff --git a/fizz-bootstrap/pom.xml b/fizz-bootstrap/pom.xml index 5d39cf8..4b11a4c 100644 --- a/fizz-bootstrap/pom.xml +++ b/fizz-bootstrap/pom.xml @@ -87,11 +87,13 @@ sh boot.sh + boot.cmd true src/main/resources application.yml + bootstrap.yml log4j2-spring.xml true diff --git a/fizz-bootstrap/sh/boot.cmd b/fizz-bootstrap/sh/boot.cmd new file mode 100644 index 0000000..0e6051e --- /dev/null +++ b/fizz-bootstrap/sh/boot.cmd @@ -0,0 +1,69 @@ +@echo off + +if not exist "%JAVA_HOME%\bin\java.exe" echo Please set the JAVA_HOME variable in your environment. & EXIT /B 1 + +setlocal enabledelayedexpansion + +set BASE_DIR=%~dp0 +set APOLLO_META_SERVER=http://localhost:66 +set APOLLO_ENV=prod +set SPRING_PROFILES_ACTIVE=prod +set APP_NAME=fizz-gateway-community +set APP_NAME_JAR=%APP_NAME%.jar +set APP_DEP_DIR=%BASE_DIR% +set APP_LOG_DIR=%APP_DEP_DIR%logs +set JAVA_CMD="%JAVA_HOME%\bin\java.exe" + +IF NOT EXIST "%APP_LOG_DIR%" ( + MKDIR "%APP_LOG_DIR%" +) + +if "%1" == "stop" goto stop +if "%1" == "status" goto status + +if not "%1" == "start" if not "%1" == "restart" ( + echo "Usage: %0 {start|stop|restart|status}" + goto end +) + +rem JVM CONFIG +set MEM_OPTS=-Xms256m -Xmx4096m -XX:MetaspaceSize=128m -XX:MaxMetaspaceSize=256m + +set JAVA_OPTS=%MEM_OPTS% -XX:+AggressiveOpts -XX:+UseBiasedLocking -XX:+UseG1GC -XX:+HeapDumpOnOutOfMemoryError -XX:-OmitStackTraceInFastThrow -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintHeapAtGC -Xloggc:"%APP_LOG_DIR%\%START_DATE_TIME%.gc" -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath="%APP_LOG_DIR%\dump.logs" -Dreactor.netty.pool.maxIdleTime=120000 -Dorg.jboss.netty.epollBugWorkaround=true + +if "%1" == "start" goto start +if "%1" == "restart" goto stop + +:status +for /f "tokens=1" %%i in ('jps -m ^| find /I "%APP_NAME%"') do ( + echo "%APP_NAME_JAR% is runing" + goto end +) +echo "%APP_NAME_JAR% is not runing" +goto end + +:stop +for /f "tokens=1" %%i in ('jps -m ^| find /I "%APP_NAME%"') do ( + goto doStop +) +echo "WARN:%APP_NAME_JAR% is not runing" +if "%1" == "restart" goto start +goto end + +:doStop +echo "Stoping %APP_NAME_JAR%..." +for /f "tokens=1" %%i in ('jps -m ^| find "%APP_NAME%"') do ( taskkill /F /PID %%i ) +if "%1" == "restart" goto start +goto end + +:start +for /f "tokens=1" %%i in ('jps -m ^| find /I "%APP_NAME%"') do ( + echo "WARN:%APP_NAME_JAR% already started! Ignoring startup request." + goto end +) + +echo "Starting %APP_NAME_JAR% ..." +%JAVA_CMD% %JAVA_OPTS% -Dfile.encoding=UTF-8 -Dspring.profiles.active=%SPRING_PROFILES_ACTIVE% -Denv=%APOLLO_ENV% -Dapollo.meta=%APOLLO_META_SERVER% -Dlogging.config=log4j2-spring.xml -jar "%APP_DEP_DIR%%APP_NAME_JAR%" +goto end + +:end diff --git a/fizz-bootstrap/src/main/resources/application.yml b/fizz-bootstrap/src/main/resources/application.yml index e9b925e..0739244 100644 --- a/fizz-bootstrap/src/main/resources/application.yml +++ b/fizz-bootstrap/src/main/resources/application.yml @@ -57,7 +57,7 @@ aggregate: # need replace password: 123456 #please input the redis password (default:123456) # need replace - database: 10 #please input the redis database (default:9) + database: 9 #please input the redis database (default:9) proxy-webclient: name: proxy aggr-webclient: From 3271575ac4a9c35f5889d85c9e5616477db54aa3 Mon Sep 17 00:00:00 2001 From: hongqiaowei Date: Wed, 23 Jun 2021 19:35:06 +0800 Subject: [PATCH 05/25] Support appid level flow control #212 --- .../src/main/java/we/util/NetworkUtils.java | 2 +- fizz-common/src/main/java/we/util/Utils.java | 22 ++ .../java/we/config/FlowStatSchedConfig.java | 189 ++++++++++++------ .../src/main/java/we/config/SystemConfig.java | 2 + .../we/controller/FlowControlController.java | 4 +- .../java/we/filter/FlowControlFilter.java | 162 +++++++++++---- .../ratelimit/ResourceRateLimitConfig.java | 128 +++++++++++- .../ResourceRateLimitConfigService.java | 81 +++++++- .../we/filter/FlowControlFilterTests.java | 32 ++- .../ResourceRateLimitConfigServiceTests.java | 11 +- .../ResourceRateLimitConfigTests.java | 27 +++ 11 files changed, 532 insertions(+), 128 deletions(-) diff --git a/fizz-common/src/main/java/we/util/NetworkUtils.java b/fizz-common/src/main/java/we/util/NetworkUtils.java index dc6a936..01a151b 100644 --- a/fizz-common/src/main/java/we/util/NetworkUtils.java +++ b/fizz-common/src/main/java/we/util/NetworkUtils.java @@ -42,7 +42,7 @@ public class NetworkUtils { private static String serverIp; - private static final String SERVER_IP = "SERVER_IP"; + private static final String SERVER_IP = "SERVER_IP"; public static String getServerIp() { try { diff --git a/fizz-common/src/main/java/we/util/Utils.java b/fizz-common/src/main/java/we/util/Utils.java index 8b64755..1d9976c 100644 --- a/fizz-common/src/main/java/we/util/Utils.java +++ b/fizz-common/src/main/java/we/util/Utils.java @@ -71,6 +71,28 @@ public abstract class Utils { b.append(k).append(c).append(v).append(separator); } + public static String extract(String str, char separator, int nx) { + int begin = 0, end = 0, n = 0, ny = nx + 1, l = str.length(); + for (int i = 0; i < l; i++) { + char c = str.charAt(i); + if (c == separator) { + n++; + if (n == nx) { + begin = i + 1; + } else if (n == ny) { + end = i; + break; + } + } + } + if (begin == 0) { + return Constants.Symbol.EMPTY; + } else if (end == 0) { + end = l; + } + return str.substring(begin, end); + } + public static String initials2lowerCase(String s) { if (StringUtils.isBlank(s)) { return s; diff --git a/fizz-core/src/main/java/we/config/FlowStatSchedConfig.java b/fizz-core/src/main/java/we/config/FlowStatSchedConfig.java index 017322a..2ab16df 100644 --- a/fizz-core/src/main/java/we/config/FlowStatSchedConfig.java +++ b/fizz-core/src/main/java/we/config/FlowStatSchedConfig.java @@ -68,6 +68,13 @@ public class FlowStatSchedConfig extends SchedConfig { private static final String _minRespTime = "\"minRespTime\":"; private static final String _maxRespTime = "\"maxRespTime\":"; + private static final String _app = "\"app\":"; + private static final String _sourceIp = "\"sourceIp\":"; + private static final String _service = "\"service\":"; + private static final String _path = "\"path\":"; + + private static final String parentResourceList = "$prl"; + @Resource private FlowStatSchedConfigProperties flowStatSchedConfigProperties; @@ -85,7 +92,7 @@ public class FlowStatSchedConfig extends SchedConfig { private long startTimeSlot = 0; - private Map key2totalBlockMap = new HashMap<>(); + private Map resourceTimeWindow2totalBlockRequestsMap = new HashMap<>(128); @Scheduled(cron = "${flow-stat-sched.cron}") public void sched() { @@ -105,78 +112,139 @@ public class FlowStatSchedConfig extends SchedConfig { return; } - key2totalBlockMap.clear(); + resourceTimeWindow2totalBlockRequestsMap.clear(); resourceTimeWindowStats.forEach(rtws -> { + String resource = rtws.getResourceId(); List wins = rtws.getWindows(); wins.forEach(w -> { - AtomicLong totalBlock = key2totalBlockMap.computeIfAbsent(String.format("%s%s", - ResourceRateLimitConfig.GLOBAL, w.getStartTime()), key -> new AtomicLong(0)); - totalBlock.addAndGet(w.getBlockRequests()); + long t = w.getStartTime(); + long blockRequests = w.getBlockRequests(); + resourceTimeWindow2totalBlockRequestsMap.put(resource + t, new AtomicLong(blockRequests)); + }); + }); + + resourceTimeWindowStats.forEach(rtws -> { + String resource = rtws.getResourceId(); + List wins = rtws.getWindows(); + wins.forEach(w -> { + accumulateParents(resource, w.getStartTime(), w.getBlockRequests()); }); }); resourceTimeWindowStats.forEach( rtws -> { String resource = rtws.getResourceId(); - ResourceRateLimitConfig config = resourceRateLimitConfigService.getResourceRateLimitConfig(resource); - int id = (config == null ? 0 : config.id); - int type; - if (ResourceRateLimitConfig.GLOBAL.equals(resource)) { - type = ResourceRateLimitConfig.Type.GLOBAL; - } else if (resource.charAt(0) == '/') { - type = ResourceRateLimitConfig.Type.API; - } else { - type = ResourceRateLimitConfig.Type.SERVICE; - } - List wins = rtws.getWindows(); - wins.forEach( - w -> { - StringBuilder b = ThreadContext.getStringBuilder(); - Long winStart = w.getStartTime(); - BigDecimal rps = w.getRps(); - double qps; - if (rps == null) { - qps = 0.00; - } else { - qps = rps.doubleValue(); - } - - AtomicLong totalBlock = key2totalBlockMap.get(String.format("%s%s", resource, winStart)); - Long totalBlockReqs = totalBlock != null ? totalBlock.get() : w.getBlockRequests(); - - b.append(Constants.Symbol.LEFT_BRACE); - b.append(_ip); toJsonStringValue(b, ip); b.append(Constants.Symbol.COMMA); - b.append(_id); b.append(id); b.append(Constants.Symbol.COMMA); - b.append(_resource); toJsonStringValue(b, resource); b.append(Constants.Symbol.COMMA); - b.append(_type); b.append(type); b.append(Constants.Symbol.COMMA); - b.append(_start); b.append(winStart); b.append(Constants.Symbol.COMMA); - b.append(_reqs); b.append(w.getTotal()); b.append(Constants.Symbol.COMMA); - b.append(_completeReqs); b.append(w.getCompReqs()); b.append(Constants.Symbol.COMMA); - b.append(_peakConcurrents); b.append(w.getPeakConcurrentReqeusts()); b.append(Constants.Symbol.COMMA); - b.append(_reqPerSec); b.append(qps); b.append(Constants.Symbol.COMMA); - b.append(_blockReqs); b.append(w.getBlockRequests()); b.append(Constants.Symbol.COMMA); - b.append(_totalBlockReqs); b.append(totalBlockReqs); b.append(Constants.Symbol.COMMA); - b.append(_errors); b.append(w.getErrors()); b.append(Constants.Symbol.COMMA); - b.append(_avgRespTime); b.append(w.getAvgRt()); b.append(Constants.Symbol.COMMA); - b.append(_maxRespTime); b.append(w.getMax()); b.append(Constants.Symbol.COMMA); - b.append(_minRespTime); b.append(w.getMin()); - b.append(Constants.Symbol.RIGHT_BRACE); - String msg = b.toString(); - if ("kafka".equals(flowStatSchedConfigProperties.getDest())) { // for internal use - log.warn(msg, LogService.HANDLE_STGY, LogService.toKF(flowStatSchedConfigProperties.getQueue())); - } else { - rt.convertAndSend(flowStatSchedConfigProperties.getQueue(), msg).subscribe(); - } - if (log.isDebugEnabled()) { - log.debug("report " + toDP19(winStart) + " win10: " + msg); - } + String app = null, pi = null, node = ResourceRateLimitConfig.NODE, service = null, path = null; + int type = ResourceRateLimitConfig.Type.NODE, id = 0; + ResourceRateLimitConfig c = resourceRateLimitConfigService.getResourceRateLimitConfig(resource); + if (c == null) { + service = ResourceRateLimitConfig.getService(resource); + if (service != null) { + type = ResourceRateLimitConfig.Type.SERVICE_DEFAULT; + } else { + app = ResourceRateLimitConfig.getApp(resource); + if (app != null) { + type = ResourceRateLimitConfig.Type.APP_DEFAULT; } - ); + } + } else { + app = c.app; + pi = c.ip; + service = c.service; + path = c.path; + type = c.type; + id = c.id; + } + + List wins = rtws.getWindows(); + for (int i = 0; i < wins.size(); i++) { + TimeWindowStat w = wins.get(i); + StringBuilder b = ThreadContext.getStringBuilder(); + long timeWin = w.getStartTime(); + BigDecimal rps = w.getRps(); + double qps; + if (rps == null) { + qps = 0.00; + } else { + qps = rps.doubleValue(); + } + + AtomicLong totalBlockRequests = resourceTimeWindow2totalBlockRequestsMap.get(resource + timeWin); + long tbrs = (totalBlockRequests == null ? w.getBlockRequests() : totalBlockRequests.longValue()); + + b.append(Constants.Symbol.LEFT_BRACE); + b.append(_ip); toJsonStringValue(b, ip); b.append(Constants.Symbol.COMMA); + b.append(_id); b.append(id); b.append(Constants.Symbol.COMMA); + + String r = null; + if (type == ResourceRateLimitConfig.Type.NODE) { + r = ResourceRateLimitConfig.NODE; + } else if (type == ResourceRateLimitConfig.Type.SERVICE_DEFAULT || type == ResourceRateLimitConfig.Type.SERVICE) { + r = service; + } + if (r != null) { + b.append(_resource); toJsonStringValue(b, r); b.append(Constants.Symbol.COMMA); + } + + b.append(_type); b.append(type); b.append(Constants.Symbol.COMMA); + + if (app != null) { + b.append(_app); toJsonStringValue(b, app); b.append(Constants.Symbol.COMMA); + } + + if (pi != null) { + b.append(_sourceIp); toJsonStringValue(b, pi); b.append(Constants.Symbol.COMMA); + } + + if (service != null) { + b.append(_service); toJsonStringValue(b, service); b.append(Constants.Symbol.COMMA); + } + + if (path != null) { + b.append(_path); toJsonStringValue(b, path); b.append(Constants.Symbol.COMMA); + } + + b.append(_start); b.append(timeWin); b.append(Constants.Symbol.COMMA); + b.append(_reqs); b.append(w.getTotal()); b.append(Constants.Symbol.COMMA); + b.append(_completeReqs); b.append(w.getCompReqs()); b.append(Constants.Symbol.COMMA); + b.append(_peakConcurrents); b.append(w.getPeakConcurrentReqeusts()); b.append(Constants.Symbol.COMMA); + b.append(_reqPerSec); b.append(qps); b.append(Constants.Symbol.COMMA); + b.append(_blockReqs); b.append(w.getBlockRequests()); b.append(Constants.Symbol.COMMA); + b.append(_totalBlockReqs); b.append(tbrs); b.append(Constants.Symbol.COMMA); + b.append(_errors); b.append(w.getErrors()); b.append(Constants.Symbol.COMMA); + b.append(_avgRespTime); b.append(w.getAvgRt()); b.append(Constants.Symbol.COMMA); + b.append(_maxRespTime); b.append(w.getMax()); b.append(Constants.Symbol.COMMA); + b.append(_minRespTime); b.append(w.getMin()); + b.append(Constants.Symbol.RIGHT_BRACE); + String msg = b.toString(); + if ("kafka".equals(flowStatSchedConfigProperties.getDest())) { // for internal use + log.warn(msg, LogService.HANDLE_STGY, LogService.toKF(flowStatSchedConfigProperties.getQueue())); + } else { + rt.convertAndSend(flowStatSchedConfigProperties.getQueue(), msg).subscribe(); + } + if (log.isDebugEnabled()) { + log.debug("report " + toDP19(timeWin) + " win10: " + msg); + } + } } ); startTimeSlot = recentEndTimeSlot; - log.info(toDP23(st) + " fss " + toDP23(System.currentTimeMillis())); + if (log.isInfoEnabled()) { + log.info(toDP23(st) + " fss " + toDP23(System.currentTimeMillis())); + } + } + + private void accumulateParents(String resource, long timeWin, long blockRequests) { + List prl = ThreadContext.getArrayList(parentResourceList, String.class); + resourceRateLimitConfigService.getParentsTo(resource, prl); + for (int i = 0; i < prl.size(); i++) { + String parentResource = prl.get(i); + AtomicLong parentTotalBlockRequests = resourceTimeWindow2totalBlockRequestsMap.get(parentResource + timeWin); + if (parentTotalBlockRequests != null) { + parentTotalBlockRequests.addAndGet(blockRequests); + } + } } private long getRecentEndTimeSlot(FlowStat flowStat) { @@ -198,8 +266,7 @@ public class FlowStatSchedConfig extends SchedConfig { } else { interval = 0; } - long recentEndTimeSlot = currentTimeSlot - interval * 1000 - 10 * 1000; - return recentEndTimeSlot; + return currentTimeSlot - interval * 1000 - 10 * 1000; } private String toDP19(long startTimeSlot) { diff --git a/fizz-core/src/main/java/we/config/SystemConfig.java b/fizz-core/src/main/java/we/config/SystemConfig.java index 2c559b8..e3fbe53 100644 --- a/fizz-core/src/main/java/we/config/SystemConfig.java +++ b/fizz-core/src/main/java/we/config/SystemConfig.java @@ -50,6 +50,8 @@ public class SystemConfig { public static final String DEFAULT_GATEWAY_TEST_PREFIX = "/_proxytest"; + public static final String DEFAULT_GATEWAY_TEST = "_proxytest"; + public static final String DEFAULT_GATEWAY_TEST_PREFIX0 = "/_proxytest/"; private String gatewayPrefix = DEFAULT_GATEWAY_PREFIX; diff --git a/fizz-core/src/main/java/we/controller/FlowControlController.java b/fizz-core/src/main/java/we/controller/FlowControlController.java index 0ea8944..780e777 100644 --- a/fizz-core/src/main/java/we/controller/FlowControlController.java +++ b/fizz-core/src/main/java/we/controller/FlowControlController.java @@ -72,11 +72,11 @@ public class FlowControlController { long currentTimeSlot = flowStat.currentTimeSlotId(); long startTimeSlot = currentTimeSlot - recent * 1000; TimeWindowStat timeWindowStat = null; - List wins = flowStat.getResourceTimeWindowStats(ResourceRateLimitConfig.GLOBAL, startTimeSlot, currentTimeSlot, recent); + List wins = flowStat.getResourceTimeWindowStats(ResourceRateLimitConfig.NODE_RESOURCE, startTimeSlot, currentTimeSlot, recent); if (wins == null || wins.isEmpty()) { result.put("rps", 0); } else { - concurrents = flowStat.getConcurrentRequests(ResourceRateLimitConfig.GLOBAL); + concurrents = flowStat.getConcurrentRequests(ResourceRateLimitConfig.NODE_RESOURCE); result.put("concurrents", concurrents); timeWindowStat = wins.get(0).getWindows().get(0); BigDecimal winrps = timeWindowStat.getRps(); diff --git a/fizz-core/src/main/java/we/filter/FlowControlFilter.java b/fizz-core/src/main/java/we/filter/FlowControlFilter.java index 37f2832..55a66fa 100644 --- a/fizz-core/src/main/java/we/filter/FlowControlFilter.java +++ b/fizz-core/src/main/java/we/filter/FlowControlFilter.java @@ -22,6 +22,7 @@ import java.util.List; import javax.annotation.Resource; +import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -35,7 +36,11 @@ import org.springframework.web.server.WebFilterChain; import reactor.core.publisher.Mono; import reactor.core.publisher.SignalType; +import we.config.SystemConfig; import we.flume.clients.log4j2appender.LogService; +import we.legacy.RespEntity; +import we.plugin.auth.ApiConfigService; +import we.plugin.auth.AppService; import we.stats.BlockType; import we.stats.FlowStat; import we.stats.IncrRequestResult; @@ -43,6 +48,8 @@ import we.stats.ResourceConfig; import we.stats.ratelimit.ResourceRateLimitConfig; import we.stats.ratelimit.ResourceRateLimitConfigService; import we.util.Constants; +import we.util.JacksonUtils; +import we.util.ThreadContext; import we.util.WebUtils; /** @@ -69,10 +76,15 @@ public class FlowControlFilter extends FizzWebFilter { @Resource private ResourceRateLimitConfigService resourceRateLimitConfigService; - // @Resource @Autowired(required = false) private FlowStat flowStat; + @Resource + private ApiConfigService apiConfigService; + + @Resource + private AppService appService; + @Override public Mono doFilter(ServerWebExchange exchange, WebFilterChain chain) { @@ -81,61 +93,60 @@ public class FlowControlFilter extends FizzWebFilter { if (secFS == -1) { return WebUtils.responseError(exchange, HttpStatus.INTERNAL_SERVER_ERROR.value(), "request path should like /optional-prefix/service-name/real-biz-path"); } - String svc = path.substring(1, secFS); - boolean adminReq = false; - if (svc.equals(admin) || svc.equals(actuator)) { + String service = path.substring(1, secFS); + boolean adminReq = false, proxyTestReq = false; + if (service.equals(admin) || service.equals(actuator)) { adminReq = true; exchange.getAttributes().put(ADMIN_REQUEST, Constants.Symbol.EMPTY); + } else if (service.equals(SystemConfig.DEFAULT_GATEWAY_TEST)) { + proxyTestReq = true; + } else { + service = WebUtils.getClientService(exchange); } - if (flowControlFilterProperties.isFlowControl() && !adminReq) { - String service = WebUtils.getClientService(exchange); -// String reqPath = WebUtils.getClientReqPath(exchange); + if (flowControlFilterProperties.isFlowControl() && !adminReq && !proxyTestReq) { + LogService.setBizId(exchange.getRequest().getId()); + if (!apiConfigService.serviceConfigMap.containsKey(service)) { + String json = RespEntity.toJson(HttpStatus.FORBIDDEN.value(), "no service " + service, exchange.getRequest().getId()); + return WebUtils.buildJsonDirectResponse(exchange, HttpStatus.FORBIDDEN, null, json); + } + String app = WebUtils.getAppId(exchange); + if (app != null && !appService.getAppMap().containsKey(app)) { + String json = RespEntity.toJson(HttpStatus.FORBIDDEN.value(), "no app " + app, exchange.getRequest().getId()); + return WebUtils.buildJsonDirectResponse(exchange, HttpStatus.FORBIDDEN, null, json); + } + path = WebUtils.getClientReqPath(exchange); + String ip = WebUtils.getOriginIp(exchange); + long currentTimeSlot = flowStat.currentTimeSlotId(); - ResourceRateLimitConfig globalConfig = resourceRateLimitConfigService - .getResourceRateLimitConfig(ResourceRateLimitConfig.GLOBAL); - ResourceRateLimitConfig serviceConfig = resourceRateLimitConfigService.getResourceRateLimitConfig(service); - if (serviceConfig == null) { - serviceConfig = resourceRateLimitConfigService - .getResourceRateLimitConfig(ResourceRateLimitConfig.SERVICE_DEFAULT); - } - - // global - List resourceConfigs = new ArrayList<>(); - ResourceConfig globalResCfg = new ResourceConfig(ResourceRateLimitConfig.GLOBAL, 0, 0); - if (globalConfig != null && globalConfig.isEnable()) { - globalResCfg.setMaxCon(globalConfig.concurrents); - globalResCfg.setMaxQPS(globalConfig.qps); - } - resourceConfigs.add(globalResCfg); - - // service - ResourceConfig serviceResCfg = new ResourceConfig(service, 0, 0); - if (serviceConfig != null && serviceConfig.isEnable()) { - serviceResCfg.setMaxCon(serviceConfig.concurrents); - serviceResCfg.setMaxQPS(serviceConfig.qps); - } - resourceConfigs.add(serviceResCfg); - + List resourceConfigs = getFlowControlConfigs(app, ip, null, service, path); IncrRequestResult result = flowStat.incrRequest(resourceConfigs, currentTimeSlot); if (result != null && !result.isSuccess()) { + String blockedResourceId = result.getBlockedResourceId(); if (BlockType.CONCURRENT_REQUEST == result.getBlockType()) { - log.info("exceed {} flow limit, blocked by maximum concurrent requests", - result.getBlockedResourceId(), LogService.BIZ_ID, exchange.getRequest().getId()); + log.info("exceed {} flow limit, blocked by maximum concurrent requests", blockedResourceId, LogService.BIZ_ID, exchange.getRequest().getId()); } else { - log.info("exceed {} flow limit, blocked by maximum QPS", result.getBlockedResourceId(), - LogService.BIZ_ID, exchange.getRequest().getId()); + log.info("exceed {} flow limit, blocked by maximum QPS", blockedResourceId, LogService.BIZ_ID, exchange.getRequest().getId()); } -// ResourceRateLimitConfig config = result.getBlockedResourceId().equals(globalConfig.resource) -// ? globalConfig -// : serviceConfig; + ResourceRateLimitConfig c = resourceRateLimitConfigService.getResourceRateLimitConfig(ResourceRateLimitConfig.NODE_RESOURCE); + String rt = c.responseType, rc = c.responseContent; + c = resourceRateLimitConfigService.getResourceRateLimitConfig(blockedResourceId); + if (c != null) { + if (StringUtils.isNotBlank(c.responseType)) { + rt = c.responseType; + } + if (StringUtils.isNotBlank(c.responseContent)) { + rc = c.responseContent; + } + } ServerHttpResponse resp = exchange.getResponse(); resp.setStatusCode(HttpStatus.OK); - resp.getHeaders().add(HttpHeaders.CONTENT_TYPE, globalConfig.responseType); - return resp.writeWith(Mono.just(resp.bufferFactory().wrap(globalConfig.responseContent.getBytes()))); + resp.getHeaders().add(HttpHeaders.CONTENT_TYPE, rt); + return resp.writeWith(Mono.just(resp.bufferFactory().wrap(rc.getBytes()))); + } else { long start = System.currentTimeMillis(); return chain.filter(exchange).doFinally(s -> { @@ -151,4 +162,73 @@ public class FlowControlFilter extends FizzWebFilter { return chain.filter(exchange); } + + private List getFlowControlConfigs(String app, String ip, String node, String service, String path) { + if (log.isDebugEnabled()) { + log.debug("get flow control config by app={}, ip={}, node={}, service={}, path={}", app, ip, node, service, path); + } + List resourceConfigs = new ArrayList<>(9); + StringBuilder b = ThreadContext.getStringBuilder(); + + checkRateLimitConfigAndAddTo(resourceConfigs, b, null, null, ResourceRateLimitConfig.NODE, null, null, null); + checkRateLimitConfigAndAddTo(resourceConfigs, b, null, null, null, service, null, ResourceRateLimitConfig.SERVICE_DEFAULT); + checkRateLimitConfigAndAddTo(resourceConfigs, b, null, null, null, service, path, null); + + if (app != null) { + checkRateLimitConfigAndAddTo(resourceConfigs, b, app, null, null, null, null, ResourceRateLimitConfig.APP_DEFAULT); + checkRateLimitConfigAndAddTo(resourceConfigs, b, app, null, null, service, null, null); + checkRateLimitConfigAndAddTo(resourceConfigs, b, app, null, null, service, path, null); + } + + if (ip != null) { + checkRateLimitConfigAndAddTo(resourceConfigs, b, null, ip, null, null, null, null); + checkRateLimitConfigAndAddTo(resourceConfigs, b, null, ip, null, service, null, null); + checkRateLimitConfigAndAddTo(resourceConfigs, b, null, ip, null, service, path, null); + } + + if (log.isDebugEnabled()) { + log.debug("resource configs: " + JacksonUtils.writeValueAsString(resourceConfigs)); + } + return resourceConfigs; + } + + private void checkRateLimitConfigAndAddTo(List resourceConfigs, StringBuilder b, String app, String ip, String node, String service, String path, String defaultRateLimitConfigId) { + ResourceRateLimitConfig.buildResourceIdTo(b, app, ip, node, service, path); + String resourceId = b.toString(); + checkRateLimitConfigAndAddTo(resourceConfigs, resourceId, defaultRateLimitConfigId); + b.delete(0, b.length()); + } + + private void checkRateLimitConfigAndAddTo(List resourceConfigs, String resource, String defaultRateLimitConfigId) { + ResourceConfig rc = null; + ResourceRateLimitConfig rateLimitConfig = resourceRateLimitConfigService.getResourceRateLimitConfig(resource); + if (rateLimitConfig != null && rateLimitConfig.isEnable()) { + rc = new ResourceConfig(resource, rateLimitConfig.concurrents, rateLimitConfig.qps); + resourceConfigs.add(rc); + } else { + String node = ResourceRateLimitConfig.getNode(resource); + if (node != null && node.equals(ResourceRateLimitConfig.NODE)) { + rc = new ResourceConfig(resource, 0, 0); + } + if (defaultRateLimitConfigId != null) { + if (defaultRateLimitConfigId.equals(ResourceRateLimitConfig.SERVICE_DEFAULT)) { + rc = new ResourceConfig(resource, 0, 0); + rateLimitConfig = resourceRateLimitConfigService.getResourceRateLimitConfig(ResourceRateLimitConfig.SERVICE_DEFAULT_RESOURCE); + if (rateLimitConfig != null && rateLimitConfig.isEnable()) { + rc.setMaxCon(rateLimitConfig.concurrents); + rc.setMaxQPS(rateLimitConfig.qps); + } + } + if (defaultRateLimitConfigId.equals(ResourceRateLimitConfig.APP_DEFAULT)) { + rateLimitConfig = resourceRateLimitConfigService.getResourceRateLimitConfig(ResourceRateLimitConfig.APP_DEFAULT_RESOURCE); + if (rateLimitConfig != null && rateLimitConfig.isEnable()) { + rc = new ResourceConfig(resource, rateLimitConfig.concurrents, rateLimitConfig.qps); + } + } + } + if (rc != null) { + resourceConfigs.add(rc); + } + } + } } diff --git a/fizz-core/src/main/java/we/stats/ratelimit/ResourceRateLimitConfig.java b/fizz-core/src/main/java/we/stats/ratelimit/ResourceRateLimitConfig.java index 596538a..58a499c 100644 --- a/fizz-core/src/main/java/we/stats/ratelimit/ResourceRateLimitConfig.java +++ b/fizz-core/src/main/java/we/stats/ratelimit/ResourceRateLimitConfig.java @@ -17,7 +17,10 @@ package we.stats.ratelimit; +import com.fasterxml.jackson.annotation.JsonIgnore; +import we.util.Constants; import we.util.JacksonUtils; +import we.util.Utils; /** * @author hongqiaowei @@ -26,30 +29,51 @@ import we.util.JacksonUtils; public class ResourceRateLimitConfig { public static interface Type { - static final byte GLOBAL = 1; + static final byte NODE = 1; static final byte SERVICE_DEFAULT = 2; static final byte SERVICE = 3; static final byte API = 4; + static final byte APP_DEFAULT = 5; + static final byte APP = 6; + static final byte IP = 7; } - public static final int DELETED = 1; + public static final int DELETED = 1; - public static final String GLOBAL = "_global"; + public static final String NODE = "_global"; - public static final String SERVICE_DEFAULT = "service_default"; + public static final String NODE_RESOURCE = buildResourceId(null, null, NODE, null, null); - private static final int ENABLE = 1; + public static final String SERVICE_DEFAULT = "service_default"; - private static final int UNABLE = 0; + public static final String SERVICE_DEFAULT_RESOURCE = buildResourceId(null, null, null, SERVICE_DEFAULT, null); + + public static final String APP_DEFAULT = "app_default"; + + public static final String APP_DEFAULT_RESOURCE = buildResourceId(APP_DEFAULT, null, null, null, null); + + private static final int ENABLE = 1; + + private static final int UNABLE = 0; public int isDeleted = 0; public int id; - private boolean enable = true; + private boolean enable = true; public String resource; + public String service; + + public String path; + + public String app; + + public String ip; + + public String node; + public byte type; public long qps; @@ -72,6 +96,96 @@ public class ResourceRateLimitConfig { } } + public void setResource(String r) { + resource = r; + if (!resource.equals(NODE)) { + service = resource; + } + } + + public void setType(byte t) { + type = t; + if (type == Type.NODE) { + node = NODE; + } else if (type == Type.SERVICE_DEFAULT) { + service = SERVICE_DEFAULT; + } else if (type == Type.APP_DEFAULT) { + app = APP_DEFAULT; + } + } + + private String resourceId = null; + + @JsonIgnore + public String getResourceId() { + if (resourceId == null) { + resourceId = + (app == null ? "" : app) + '@' + + (ip == null ? "" : ip) + '@' + + (node == null ? "" : node) + '@' + + (service == null ? "" : service) + '@' + + (path == null ? "" : path) + ; + } + return resourceId; + } + + public static String buildResourceId(String app, String ip, String node, String service, String path) { + StringBuilder b = new StringBuilder(32); + buildResourceIdTo(b, app, ip, node, service, path); + return b.toString(); + } + + public static void buildResourceIdTo(StringBuilder b, String app, String ip, String node, String service, String path) { + b.append(app == null ? Constants.Symbol.EMPTY : app) .append(Constants.Symbol.AT); + b.append(ip == null ? Constants.Symbol.EMPTY : ip) .append(Constants.Symbol.AT); + b.append(node == null ? Constants.Symbol.EMPTY : node) .append(Constants.Symbol.AT); + b.append(service == null ? Constants.Symbol.EMPTY : service) .append(Constants.Symbol.AT); + b.append(path == null ? Constants.Symbol.EMPTY : path); + } + + public static String getApp(String resource) { + int i = resource.indexOf(Constants.Symbol.AT); + if (i == 0) { + return null; + } else { + return resource.substring(0, i); + } + } + + public static String getIp(String resource) { + String extract = Utils.extract(resource, Constants.Symbol.AT, 1); + if (extract.equals(Constants.Symbol.EMPTY)) { + return null; + } + return extract; + } + + public static String getNode(String resource) { + String extract = Utils.extract(resource, Constants.Symbol.AT, 2); + if (extract.equals(Constants.Symbol.EMPTY)) { + return null; + } + return extract; + } + + public static String getService(String resource) { + String extract = Utils.extract(resource, Constants.Symbol.AT, 3); + if (extract.equals(Constants.Symbol.EMPTY)) { + return null; + } + return extract; + } + + public static String getPath(String resource) { + int i = resource.lastIndexOf(Constants.Symbol.AT); + if (i == resource.length() - 1) { + return null; + } else { + return resource.substring(i); + } + } + @Override public String toString() { return JacksonUtils.writeValueAsString(this); diff --git a/fizz-core/src/main/java/we/stats/ratelimit/ResourceRateLimitConfigService.java b/fizz-core/src/main/java/we/stats/ratelimit/ResourceRateLimitConfigService.java index 9d84f9b..6f046da 100644 --- a/fizz-core/src/main/java/we/stats/ratelimit/ResourceRateLimitConfigService.java +++ b/fizz-core/src/main/java/we/stats/ratelimit/ResourceRateLimitConfigService.java @@ -27,13 +27,11 @@ import we.config.AggregateRedisConfig; import we.flume.clients.log4j2appender.LogService; import we.util.JacksonUtils; import we.util.ReactorUtils; +import we.util.ThreadContext; import javax.annotation.PostConstruct; import javax.annotation.Resource; -import java.util.AbstractMap; -import java.util.HashMap; -import java.util.Map; -import java.util.Objects; +import java.util.*; import java.util.concurrent.TimeUnit; import java.util.function.Supplier; @@ -130,7 +128,7 @@ public class ResourceRateLimitConfigService { ResourceRateLimitConfig rrlc = JacksonUtils.readValue(json, ResourceRateLimitConfig.class); ResourceRateLimitConfig r = oldResourceRateLimitConfigMap.remove(rrlc.id); if (rrlc.isDeleted != ResourceRateLimitConfig.DELETED && r != null) { - resourceRateLimitConfigMap.remove(r.resource); + resourceRateLimitConfigMap.remove(r.getResourceId()); } updateResourceRateLimitConfigMap(rrlc, resourceRateLimitConfigMap); if (rrlc.isDeleted != ResourceRateLimitConfig.DELETED) { @@ -158,11 +156,11 @@ public class ResourceRateLimitConfigService { private void updateResourceRateLimitConfigMap(ResourceRateLimitConfig rrlc, Map resourceRateLimitConfigMap) { if (rrlc.isDeleted == ResourceRateLimitConfig.DELETED) { - ResourceRateLimitConfig removedRrlc = resourceRateLimitConfigMap.remove(rrlc.resource); + ResourceRateLimitConfig removedRrlc = resourceRateLimitConfigMap.remove(rrlc.getResourceId()); log.info("remove " + removedRrlc); } else { - ResourceRateLimitConfig existRrlc = resourceRateLimitConfigMap.get(rrlc.resource); - resourceRateLimitConfigMap.put(rrlc.resource, rrlc); + ResourceRateLimitConfig existRrlc = resourceRateLimitConfigMap.get(rrlc.getResourceId()); + resourceRateLimitConfigMap.put(rrlc.getResourceId(), rrlc); if (existRrlc == null) { log.info("add " + rrlc); } else { @@ -182,4 +180,71 @@ public class ResourceRateLimitConfigService { public Map getResourceRateLimitConfigMap() { return resourceRateLimitConfigMap; } + + public void getParentsTo(String resource, List parentList) { + String app = null, ip = null, node = null, service = null, path = null; + ResourceRateLimitConfig c = resourceRateLimitConfigMap.get(resource); + if (c == null) { + service = ResourceRateLimitConfig.getService(resource); + if (service != null) { + parentList.add(ResourceRateLimitConfig.NODE_RESOURCE); + } + return; + } else { + if (c.type == ResourceRateLimitConfig.Type.NODE) { + return; + } + if (c.type == ResourceRateLimitConfig.Type.SERVICE) { + parentList.add(ResourceRateLimitConfig.NODE_RESOURCE); + return; + } + app = c.app; + ip = c.ip; + service = c.service; + path = c.path; + } + + StringBuilder b = ThreadContext.getStringBuilder(); + + if (app != null) { + if (path != null) { + ResourceRateLimitConfig.buildResourceIdTo(b, app, null, null, service, null); + checkRateLimitConfigAndAddTo(b, parentList); + ResourceRateLimitConfig.buildResourceIdTo(b, app, null, null, null, null); + checkRateLimitConfigAndAddTo(b, parentList); + } else if (service != null) { + ResourceRateLimitConfig.buildResourceIdTo(b, app, null, null, null, null); + checkRateLimitConfigAndAddTo(b, parentList); + } + } + + if (ip != null) { + if (path != null) { + ResourceRateLimitConfig.buildResourceIdTo(b, null, ip, null, service, null); + checkRateLimitConfigAndAddTo(b, parentList); + ResourceRateLimitConfig.buildResourceIdTo(b, null, ip, null, null, null); + checkRateLimitConfigAndAddTo(b, parentList); + } else if (service != null) { + ResourceRateLimitConfig.buildResourceIdTo(b, null, ip, null, null, null); + checkRateLimitConfigAndAddTo(b, parentList); + } + } + + if (path != null) { + ResourceRateLimitConfig.buildResourceIdTo(b, null, null, null, service, null); + parentList.add(b.toString()); + b.delete(0, b.length()); + } + + parentList.add(ResourceRateLimitConfig.NODE_RESOURCE); + } + + private void checkRateLimitConfigAndAddTo(StringBuilder resourceStringBuilder, List resourceList) { + String r = resourceStringBuilder.toString(); + ResourceRateLimitConfig c = resourceRateLimitConfigMap.get(r); + if (c != null) { + resourceList.add(r); + } + resourceStringBuilder.delete(0, resourceStringBuilder.length()); + } } diff --git a/fizz-core/src/test/java/we/filter/FlowControlFilterTests.java b/fizz-core/src/test/java/we/filter/FlowControlFilterTests.java index 5886e8f..e85f741 100644 --- a/fizz-core/src/test/java/we/filter/FlowControlFilterTests.java +++ b/fizz-core/src/test/java/we/filter/FlowControlFilterTests.java @@ -52,16 +52,24 @@ public class FlowControlFilterTests { // @Test void flowControlFilterTest() throws NoSuchFieldException, InterruptedException { - FlowControlFilter flowControlFilter = new FlowControlFilter(); - ReflectionUtils.set(flowControlFilter, "flowControl", true); + FlowControlFilter filter = new FlowControlFilter(); + FlowControlFilterProperties flowControlFilterProperties = new FlowControlFilterProperties(); + ReflectionUtils.set(flowControlFilterProperties, "flowControl", true); + ReflectionUtils.set(filter, "flowControlFilterProperties", flowControlFilterProperties); + FlowStat flowStat = new FlowStat(); - ReflectionUtils.set(flowControlFilter, "flowStat", flowStat); + ReflectionUtils.set(filter, "flowStat", flowStat); ResourceRateLimitConfigService resourceRateLimitConfigService = new ResourceRateLimitConfigService(); Map map = resourceRateLimitConfigService.getResourceRateLimitConfigMap(); + ResourceRateLimitConfig config = JacksonUtils.readValue("{\"concurrents\":66,\"enable\":1,\"id\":1,\"isDeleted\":0,\"resource\":\"_global\",\"type\":1}", ResourceRateLimitConfig.class); - map.put(ResourceRateLimitConfig.GLOBAL, config); - ReflectionUtils.set(flowControlFilter, "resourceRateLimitConfigService", resourceRateLimitConfigService); + map.put(ResourceRateLimitConfig.NODE_RESOURCE, config); + + config = JacksonUtils.readValue("{\"concurrents\":33,\"enable\":1,\"id\":2,\"isDeleted\":0, \"service\":\"xservice\", \"path\":\"/ypath\", \"type\":4}", ResourceRateLimitConfig.class); + map.put(config.getResourceId(), config); + + ReflectionUtils.set(filter, "resourceRateLimitConfigService", resourceRateLimitConfigService); WebTestClient client = WebTestClient.bindToWebHandler( new WebHandler() { @@ -74,14 +82,24 @@ public class FlowControlFilterTests { } } ) - .webFilter(flowControlFilter) + .webFilter(filter) .build(); + client.get().uri("/proxy/xservice/ypath").exchange(); Thread.sleep(1000); long currentTimeSlot = flowStat.currentTimeSlotId(); long startTimeSlot = currentTimeSlot - 10 * 1000; - List resourceTimeWindowStats = flowStat.getResourceTimeWindowStats("xservice", startTimeSlot, currentTimeSlot, 10); + + // System.err.println(JacksonUtils.writeValueAsString(flowStat.resourceStats)); + + String xservice = ResourceRateLimitConfig.buildResourceId(null, null, null, "xservice", null); + List resourceTimeWindowStats = flowStat.getResourceTimeWindowStats(xservice, startTimeSlot, currentTimeSlot, 10); TimeWindowStat win = resourceTimeWindowStats.get(0).getWindows().get(0); assertEquals(win.getCompReqs(), 1); + + String xserviceYpath = ResourceRateLimitConfig.buildResourceId(null, null, null, "xservice", "/ypath"); + resourceTimeWindowStats = flowStat.getResourceTimeWindowStats(xserviceYpath, startTimeSlot, currentTimeSlot, 10); + win = resourceTimeWindowStats.get(0).getWindows().get(0); + assertEquals(win.getCompReqs(), 1); } } diff --git a/fizz-core/src/test/java/we/stats/ratelimit/ResourceRateLimitConfigServiceTests.java b/fizz-core/src/test/java/we/stats/ratelimit/ResourceRateLimitConfigServiceTests.java index 8b59782..8c2bce0 100644 --- a/fizz-core/src/test/java/we/stats/ratelimit/ResourceRateLimitConfigServiceTests.java +++ b/fizz-core/src/test/java/we/stats/ratelimit/ResourceRateLimitConfigServiceTests.java @@ -9,9 +9,12 @@ import org.springframework.test.context.junit.jupiter.SpringJUnitConfig; import we.redis.RedisProperties; import we.redis.RedisServerConfiguration; import we.redis.RedisTemplateConfiguration; +import we.util.JacksonUtils; import javax.annotation.Resource; +import java.util.Map; + import static org.junit.jupiter.api.Assertions.assertEquals; /** @@ -46,11 +49,17 @@ public class ResourceRateLimitConfigServiceTests { @Test void initTest() throws Throwable { stringRedisTemplate.opsForHash().put("fizz_rate_limit", "2", "{\"concurrents\":66,\"enable\":1,\"id\":2,\"isDeleted\":0,\"resource\":\"service_default\",\"type\":2}"); + stringRedisTemplate.opsForHash().put("fizz_rate_limit", "3", "{\"concurrents\":88,\"enable\":1,\"id\":3,\"isDeleted\":0, \"type\":6, \"app\":\"xapp\", \"service\":\"yservice\" }"); resourceRateLimitConfigService.init(); - ResourceRateLimitConfig resourceRateLimitConfig = resourceRateLimitConfigService.getResourceRateLimitConfig("service_default"); + ResourceRateLimitConfig resourceRateLimitConfig = resourceRateLimitConfigService.getResourceRateLimitConfig(ResourceRateLimitConfig.SERVICE_DEFAULT_RESOURCE); + // Map resourceRateLimitConfigMap = resourceRateLimitConfigService.getResourceRateLimitConfigMap(); + // System.err.println(JacksonUtils.writeValueAsString(resourceRateLimitConfigMap)); assertEquals(resourceRateLimitConfig.concurrents, 66); // System.err.println(resourceRateLimitConfig); // Thread.currentThread().join(); + resourceRateLimitConfig = resourceRateLimitConfigService.getResourceRateLimitConfig("xapp@@@yservice@"); + assertEquals(resourceRateLimitConfig.concurrents, 88); + Thread.sleep(4000); // System.err.println("init test end"); } diff --git a/fizz-core/src/test/java/we/stats/ratelimit/ResourceRateLimitConfigTests.java b/fizz-core/src/test/java/we/stats/ratelimit/ResourceRateLimitConfigTests.java index d8fb086..80aaac2 100644 --- a/fizz-core/src/test/java/we/stats/ratelimit/ResourceRateLimitConfigTests.java +++ b/fizz-core/src/test/java/we/stats/ratelimit/ResourceRateLimitConfigTests.java @@ -17,4 +17,31 @@ public class ResourceRateLimitConfigTests { ResourceRateLimitConfig resourceRateLimitConfig = JacksonUtils.readValue(resourceRateLimitConfigJson, ResourceRateLimitConfig.class); assertEquals("application/json; charset=UTF-8", resourceRateLimitConfig.responseType); } + + @Test + void resourceIdTest() { + String resourceRateLimitConfigJson = "{\"concurrents\":1000,\"enable\":1,\"id\":1,\"isDeleted\":0,\"qps\":500, \"type\":1, \"resource\":\"_global\" }"; + ResourceRateLimitConfig c = JacksonUtils.readValue(resourceRateLimitConfigJson, ResourceRateLimitConfig.class); + String resourceId = c.getResourceId(); + assertEquals("@@_global@@", resourceId); + + String node = ResourceRateLimitConfig.getNode(resourceId); + assertEquals("_global", node); + + resourceRateLimitConfigJson = "{\"concurrents\":1000,\"enable\":1,\"id\":1,\"isDeleted\":0,\"qps\":500, \"type\":2, \"resource\":\"service_default\" }"; + c = JacksonUtils.readValue(resourceRateLimitConfigJson, ResourceRateLimitConfig.class); + resourceId = c.getResourceId(); + assertEquals("@@@service_default@", resourceId); + + resourceRateLimitConfigJson = "{\"concurrents\":1000,\"enable\":1,\"id\":1,\"isDeleted\":0,\"qps\":500, \"type\":3, \"resource\":\"xservice\" }"; + c = JacksonUtils.readValue(resourceRateLimitConfigJson, ResourceRateLimitConfig.class); + resourceId = c.getResourceId(); + assertEquals("@@@xservice@", resourceId); + + resourceId = ResourceRateLimitConfig.buildResourceId(null, null, ResourceRateLimitConfig.NODE, null, null); + assertEquals("@@_global@@", resourceId); + + resourceId = ResourceRateLimitConfig.buildResourceId(null, "192.168.1.1", null, "xservice", null); + assertEquals("@192.168.1.1@@xservice@", resourceId); + } } From 25e5c4951874a455ee665f16e77e43a0672608a2 Mon Sep 17 00:00:00 2001 From: hongqiaowei Date: Wed, 23 Jun 2021 22:17:28 +0800 Subject: [PATCH 06/25] Upgrade project dependencies #231 --- fizz-bootstrap/pom.xml | 7 +++++-- fizz-common/pom.xml | 10 +++++----- fizz-core/pom.xml | 21 ++++++++++++++++----- fizz-plugin/pom.xml | 15 ++++++++++----- fizz-spring-boot-starter/pom.xml | 17 +++++++++++------ pom.xml | 27 +++++++++++++++++---------- 6 files changed, 64 insertions(+), 33 deletions(-) diff --git a/fizz-bootstrap/pom.xml b/fizz-bootstrap/pom.xml index 4b11a4c..456cd52 100644 --- a/fizz-bootstrap/pom.xml +++ b/fizz-bootstrap/pom.xml @@ -17,11 +17,13 @@ 1.8 5.2.15.RELEASE - Dysprosium-SR20 + Dragonfruit-SR3 + Dysprosium-SR21 5.3.7.RELEASE 4.1.65.Final 4.4.14 - 2.13.3 + 2.14.1 + 1.7.31 3.12.0 1.18.20 2.7.5 @@ -29,6 +31,7 @@ 3.4.6 4.0.1 3.5.9 + 1.15 diff --git a/fizz-common/pom.xml b/fizz-common/pom.xml index 53fde5e..169398b 100644 --- a/fizz-common/pom.xml +++ b/fizz-common/pom.xml @@ -58,11 +58,6 @@ spring-session-data-redis - - commons-codec - commons-codec - - org.bouncycastle bcpkix-jdk15on @@ -98,6 +93,11 @@ + + + commons-codec + commons-codec + diff --git a/fizz-core/pom.xml b/fizz-core/pom.xml index 5e4efa7..d3d5fcc 100644 --- a/fizz-core/pom.xml +++ b/fizz-core/pom.xml @@ -37,7 +37,7 @@ org.springframework.session spring-session-bom - Dragonfruit-RELEASE + Dragonfruit-SR3 pom import @@ -153,6 +153,12 @@ com.github.ben-manes.caffeine caffeine + + + error_prone_annotations + com.google.errorprone + + @@ -253,13 +259,18 @@ - commons-codec - commons-codec + org.bouncycastle + bcpkix-jdk15on - org.bouncycastle - bcpkix-jdk15on + com.google.guava + guava + + + + commons-codec + commons-codec diff --git a/fizz-plugin/pom.xml b/fizz-plugin/pom.xml index 1022bba..a347c40 100644 --- a/fizz-plugin/pom.xml +++ b/fizz-plugin/pom.xml @@ -34,7 +34,7 @@ org.springframework.session spring-session-bom - Dragonfruit-RELEASE + Dragonfruit-SR3 pom import @@ -245,13 +245,18 @@ - commons-codec - commons-codec + org.bouncycastle + bcpkix-jdk15on - org.bouncycastle - bcpkix-jdk15on + com.google.guava + guava + + + + commons-codec + commons-codec diff --git a/fizz-spring-boot-starter/pom.xml b/fizz-spring-boot-starter/pom.xml index 6f4f739..e922344 100644 --- a/fizz-spring-boot-starter/pom.xml +++ b/fizz-spring-boot-starter/pom.xml @@ -19,7 +19,7 @@ org.springframework.session spring-session-bom - Dragonfruit-RELEASE + Dragonfruit-SR3 pom import @@ -227,11 +227,6 @@ test - - commons-codec - commons-codec - - org.bouncycastle bcpkix-jdk15on @@ -255,6 +250,16 @@ com.fizzgate fizz-plugin + + + com.google.guava + guava + + + + commons-codec + commons-codec + \ No newline at end of file diff --git a/pom.xml b/pom.xml index a008476..db4b17d 100644 --- a/pom.xml +++ b/pom.xml @@ -7,12 +7,13 @@ 2.2.13.RELEASE 5.2.15.RELEASE - Dysprosium-SR20 + Dysprosium-SR21 5.3.7.RELEASE 2.2.5.RELEASE 4.1.65.Final 4.4.14 - 2.13.3 + 2.14.1 + 1.7.31 2.7.5 1.16.1 3.4.6 @@ -86,7 +87,7 @@ org.springframework.session spring-session-bom - Dragonfruit-RELEASE + Dragonfruit-SR3 pom import @@ -129,7 +130,7 @@ io.projectreactor reactor-test - 3.3.17.RELEASE + 3.3.18.RELEASE test @@ -342,12 +343,6 @@ test - - commons-codec - commons-codec - 1.15 - - org.bouncycastle bcpkix-jdk15on @@ -359,6 +354,18 @@ kxml2 2.3.0 + + + com.google.guava + guava + 21.0 + + + + commons-codec + commons-codec + 1.15 + From a4fd860489a678e199ef97d58ca24782f4a6d923 Mon Sep 17 00:00:00 2001 From: hongqiaowei Date: Wed, 23 Jun 2021 22:29:13 +0800 Subject: [PATCH 07/25] Release v2.2.0-beta1 --- fizz-bootstrap/pom.xml | 2 +- fizz-common/pom.xml | 3 ++- fizz-core/pom.xml | 2 +- fizz-plugin/pom.xml | 2 +- fizz-spring-boot-starter/pom.xml | 2 +- pom.xml | 3 ++- 6 files changed, 8 insertions(+), 6 deletions(-) diff --git a/fizz-bootstrap/pom.xml b/fizz-bootstrap/pom.xml index 456cd52..483d2a2 100644 --- a/fizz-bootstrap/pom.xml +++ b/fizz-bootstrap/pom.xml @@ -12,7 +12,7 @@ com.fizzgate fizz-bootstrap - 2.1.0 + 2.2.0-beta1 1.8 diff --git a/fizz-common/pom.xml b/fizz-common/pom.xml index 169398b..9333253 100644 --- a/fizz-common/pom.xml +++ b/fizz-common/pom.xml @@ -5,7 +5,8 @@ fizz-gateway-community com.fizzgate - 2.1.0 + 2.2.0-beta1 + ../pom.xml 4.0.0 diff --git a/fizz-core/pom.xml b/fizz-core/pom.xml index d3d5fcc..3838841 100644 --- a/fizz-core/pom.xml +++ b/fizz-core/pom.xml @@ -5,7 +5,7 @@ fizz-gateway-community com.fizzgate - 2.1.0 + 2.2.0-beta1 ../pom.xml 4.0.0 diff --git a/fizz-plugin/pom.xml b/fizz-plugin/pom.xml index a347c40..ada2c51 100644 --- a/fizz-plugin/pom.xml +++ b/fizz-plugin/pom.xml @@ -5,7 +5,7 @@ fizz-gateway-community com.fizzgate - 2.1.0 + 2.2.0-beta1 ../pom.xml 4.0.0 diff --git a/fizz-spring-boot-starter/pom.xml b/fizz-spring-boot-starter/pom.xml index e922344..969089b 100644 --- a/fizz-spring-boot-starter/pom.xml +++ b/fizz-spring-boot-starter/pom.xml @@ -5,7 +5,7 @@ fizz-gateway-community com.fizzgate - 2.1.0 + 2.2.0-beta1 ../pom.xml 4.0.0 diff --git a/pom.xml b/pom.xml index db4b17d..57d4463 100644 --- a/pom.xml +++ b/pom.xml @@ -31,7 +31,8 @@ fizz-gateway-community ${project.artifactId} fizz gateway community - 2.1.0 + + 2.2.0-beta1 pom fizz-common From cccfd287273cd2ebddc57cacdba66a7a8f540307 Mon Sep 17 00:00:00 2001 From: hongqiaowei Date: Thu, 24 Jun 2021 16:00:32 +0800 Subject: [PATCH 08/25] Release v2.2.0-beta2 --- fizz-bootstrap/pom.xml | 4 ++-- fizz-common/pom.xml | 3 +-- fizz-core/pom.xml | 2 +- fizz-plugin/pom.xml | 2 +- fizz-spring-boot-starter/pom.xml | 2 +- pom.xml | 5 ++--- 6 files changed, 8 insertions(+), 10 deletions(-) diff --git a/fizz-bootstrap/pom.xml b/fizz-bootstrap/pom.xml index 483d2a2..e7746d6 100644 --- a/fizz-bootstrap/pom.xml +++ b/fizz-bootstrap/pom.xml @@ -12,11 +12,11 @@ com.fizzgate fizz-bootstrap - 2.2.0-beta1 + 2.2.0-beta2 1.8 - 5.2.15.RELEASE + 5.2.13.RELEASE Dragonfruit-SR3 Dysprosium-SR21 5.3.7.RELEASE diff --git a/fizz-common/pom.xml b/fizz-common/pom.xml index 9333253..8cd96fc 100644 --- a/fizz-common/pom.xml +++ b/fizz-common/pom.xml @@ -5,8 +5,7 @@ fizz-gateway-community com.fizzgate - 2.2.0-beta1 - + 2.2.0-beta2 ../pom.xml 4.0.0 diff --git a/fizz-core/pom.xml b/fizz-core/pom.xml index 3838841..a2bc0a0 100644 --- a/fizz-core/pom.xml +++ b/fizz-core/pom.xml @@ -5,7 +5,7 @@ fizz-gateway-community com.fizzgate - 2.2.0-beta1 + 2.2.0-beta2 ../pom.xml 4.0.0 diff --git a/fizz-plugin/pom.xml b/fizz-plugin/pom.xml index ada2c51..865da1d 100644 --- a/fizz-plugin/pom.xml +++ b/fizz-plugin/pom.xml @@ -5,7 +5,7 @@ fizz-gateway-community com.fizzgate - 2.2.0-beta1 + 2.2.0-beta2 ../pom.xml 4.0.0 diff --git a/fizz-spring-boot-starter/pom.xml b/fizz-spring-boot-starter/pom.xml index 969089b..f2091be 100644 --- a/fizz-spring-boot-starter/pom.xml +++ b/fizz-spring-boot-starter/pom.xml @@ -5,7 +5,7 @@ fizz-gateway-community com.fizzgate - 2.2.0-beta1 + 2.2.0-beta2 ../pom.xml 4.0.0 diff --git a/pom.xml b/pom.xml index 57d4463..95ceae9 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ 2.2.13.RELEASE - 5.2.15.RELEASE + 5.2.13.RELEASE Dysprosium-SR21 5.3.7.RELEASE 2.2.5.RELEASE @@ -31,8 +31,7 @@ fizz-gateway-community ${project.artifactId} fizz gateway community - - 2.2.0-beta1 + 2.2.0-beta2 pom fizz-common From 63eb0b52e74bd4192e1781c72c4082d9860d0059 Mon Sep 17 00:00:00 2001 From: Francis Dong Date: Fri, 25 Jun 2021 11:19:42 +0800 Subject: [PATCH 09/25] print access log of cancelled request #233 --- fizz-core/src/main/java/we/filter/FizzLogFilter.java | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/fizz-core/src/main/java/we/filter/FizzLogFilter.java b/fizz-core/src/main/java/we/filter/FizzLogFilter.java index 93e49a4..7756447 100644 --- a/fizz-core/src/main/java/we/filter/FizzLogFilter.java +++ b/fizz-core/src/main/java/we/filter/FizzLogFilter.java @@ -20,16 +20,15 @@ package we.filter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.core.annotation.Order; -import org.springframework.http.HttpStatus; import org.springframework.stereotype.Component; import org.springframework.web.server.ServerWebExchange; import org.springframework.web.server.WebFilter; import org.springframework.web.server.WebFilterChain; + import reactor.core.publisher.Mono; import we.flume.clients.log4j2appender.LogService; -import we.util.Constants; import we.util.ThreadContext; -import we.util.WebUtils; +import we.util.WebUtils; /** * @author hongqiaowei @@ -49,8 +48,8 @@ public class FizzLogFilter implements WebFilter { public Mono filter(ServerWebExchange exchange, WebFilterChain chain) { long startTime = System.currentTimeMillis(); - return chain.filter(exchange).doAfterTerminate( - () -> { + return chain.filter(exchange).doFinally( + (c) -> { if (LOGGER.isInfoEnabled()) { StringBuilder b = ThreadContext.getStringBuilder(); WebUtils.request2stringBuilder(exchange, b); From 7ef813aad56a83060371b482a7ad7fc6bb5f3ce0 Mon Sep 17 00:00:00 2001 From: Francis Dong Date: Fri, 25 Jun 2021 11:40:02 +0800 Subject: [PATCH 10/25] ON_ERROR signal and 5xx http status are classified as errors #235 --- fizz-core/src/main/java/we/filter/FlowControlFilter.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fizz-core/src/main/java/we/filter/FlowControlFilter.java b/fizz-core/src/main/java/we/filter/FlowControlFilter.java index 55a66fa..c2091c0 100644 --- a/fizz-core/src/main/java/we/filter/FlowControlFilter.java +++ b/fizz-core/src/main/java/we/filter/FlowControlFilter.java @@ -151,10 +151,10 @@ public class FlowControlFilter extends FizzWebFilter { long start = System.currentTimeMillis(); return chain.filter(exchange).doFinally(s -> { long rt = System.currentTimeMillis() - start; - if (s == SignalType.ON_COMPLETE) { - flowStat.addRequestRT(resourceConfigs, currentTimeSlot, rt, true); - } else { + if (s == SignalType.ON_ERROR || exchange.getResponse().getStatusCode().is5xxServerError()) { flowStat.addRequestRT(resourceConfigs, currentTimeSlot, rt, false); + } else { + flowStat.addRequestRT(resourceConfigs, currentTimeSlot, rt, true); } }); } From 3f4f8b5fa5cecd70beb120d40e2d4141c9c7eee2 Mon Sep 17 00:00:00 2001 From: hongqiaowei Date: Fri, 25 Jun 2021 16:04:46 +0800 Subject: [PATCH 11/25] Release v2.2.0-beta3 --- fizz-bootstrap/pom.xml | 2 +- fizz-common/pom.xml | 2 +- fizz-core/pom.xml | 2 +- .../java/we/config/FlowStatSchedConfig.java | 29 ++++++++--- .../java/we/filter/FlowControlFilter.java | 50 +++++++++++++++++++ fizz-plugin/pom.xml | 2 +- fizz-spring-boot-starter/pom.xml | 2 +- pom.xml | 2 +- 8 files changed, 79 insertions(+), 12 deletions(-) diff --git a/fizz-bootstrap/pom.xml b/fizz-bootstrap/pom.xml index e7746d6..113dac1 100644 --- a/fizz-bootstrap/pom.xml +++ b/fizz-bootstrap/pom.xml @@ -12,7 +12,7 @@ com.fizzgate fizz-bootstrap - 2.2.0-beta2 + 2.2.0-beta3 1.8 diff --git a/fizz-common/pom.xml b/fizz-common/pom.xml index 8cd96fc..c55038a 100644 --- a/fizz-common/pom.xml +++ b/fizz-common/pom.xml @@ -5,7 +5,7 @@ fizz-gateway-community com.fizzgate - 2.2.0-beta2 + 2.2.0-beta3 ../pom.xml 4.0.0 diff --git a/fizz-core/pom.xml b/fizz-core/pom.xml index a2bc0a0..936b83d 100644 --- a/fizz-core/pom.xml +++ b/fizz-core/pom.xml @@ -5,7 +5,7 @@ fizz-gateway-community com.fizzgate - 2.2.0-beta2 + 2.2.0-beta3 ../pom.xml 4.0.0 diff --git a/fizz-core/src/main/java/we/config/FlowStatSchedConfig.java b/fizz-core/src/main/java/we/config/FlowStatSchedConfig.java index 2ab16df..ec820bc 100644 --- a/fizz-core/src/main/java/we/config/FlowStatSchedConfig.java +++ b/fizz-core/src/main/java/we/config/FlowStatSchedConfig.java @@ -137,14 +137,31 @@ public class FlowStatSchedConfig extends SchedConfig { String app = null, pi = null, node = ResourceRateLimitConfig.NODE, service = null, path = null; int type = ResourceRateLimitConfig.Type.NODE, id = 0; ResourceRateLimitConfig c = resourceRateLimitConfigService.getResourceRateLimitConfig(resource); - if (c == null) { - service = ResourceRateLimitConfig.getService(resource); - if (service != null) { - type = ResourceRateLimitConfig.Type.SERVICE_DEFAULT; + + if (c == null) { // _global, service, app, ip, ip+service + node = ResourceRateLimitConfig.getNode(resource); + if (node != null && node.equals(ResourceRateLimitConfig.NODE)) { } else { + service = ResourceRateLimitConfig.getService(resource); app = ResourceRateLimitConfig.getApp(resource); - if (app != null) { - type = ResourceRateLimitConfig.Type.APP_DEFAULT; + pi = ResourceRateLimitConfig.getIp(resource); + if (service == null) { + if (app == null) { + type = ResourceRateLimitConfig.Type.IP; + } else { + ResourceRateLimitConfig appConfig = resourceRateLimitConfigService.getResourceRateLimitConfig(ResourceRateLimitConfig.APP_DEFAULT_RESOURCE); + if (appConfig != null && appConfig.isEnable()) { + type = ResourceRateLimitConfig.Type.APP_DEFAULT; + } else { + type = ResourceRateLimitConfig.Type.APP; + } + } + } else { + if (pi == null) { + type = ResourceRateLimitConfig.Type.SERVICE_DEFAULT; + } else { + type = ResourceRateLimitConfig.Type.IP; + } } } } else { diff --git a/fizz-core/src/main/java/we/filter/FlowControlFilter.java b/fizz-core/src/main/java/we/filter/FlowControlFilter.java index c2091c0..6cecf7a 100644 --- a/fizz-core/src/main/java/we/filter/FlowControlFilter.java +++ b/fizz-core/src/main/java/we/filter/FlowControlFilter.java @@ -203,6 +203,7 @@ public class FlowControlFilter extends FizzWebFilter { ResourceConfig rc = null; ResourceRateLimitConfig rateLimitConfig = resourceRateLimitConfigService.getResourceRateLimitConfig(resource); if (rateLimitConfig != null && rateLimitConfig.isEnable()) { + something4appAndIp(resourceConfigs, rateLimitConfig); rc = new ResourceConfig(resource, rateLimitConfig.concurrents, rateLimitConfig.qps); resourceConfigs.add(rc); } else { @@ -231,4 +232,53 @@ public class FlowControlFilter extends FizzWebFilter { } } } + + private void something4appAndIp(List resourceConfigs, ResourceRateLimitConfig rateLimitConfig) { + int sz = resourceConfigs.size(); + String prev = null, prevPrev = null; + if (sz > 1) { + prev = resourceConfigs.get(sz - 1).getResourceId(); + prevPrev = resourceConfigs.get(sz - 2).getResourceId(); + + if (rateLimitConfig.type == ResourceRateLimitConfig.Type.APP) { + String app = ResourceRateLimitConfig.getApp(prev); + if (app == null) { + something4(resourceConfigs, rateLimitConfig.app, null, null); + } else { + String service = ResourceRateLimitConfig.getService(prev); + if (service == null) { + } else { + app = ResourceRateLimitConfig.getApp(prevPrev); + if (app == null) { + something4(resourceConfigs, rateLimitConfig.app, null, null); + } + } + } + + } else if (rateLimitConfig.type == ResourceRateLimitConfig.Type.IP) { + + String ip = ResourceRateLimitConfig.getIp(prev); + if (ip == null) { + something4(resourceConfigs, null, rateLimitConfig.ip, null); + something4(resourceConfigs, null, rateLimitConfig.ip, rateLimitConfig.service); + } else { + String service = ResourceRateLimitConfig.getService(prev); + if (service == null) { + something4(resourceConfigs, null, rateLimitConfig.ip, rateLimitConfig.service); + } else { + ip = ResourceRateLimitConfig.getIp(prevPrev); + if (ip == null) { + something4(resourceConfigs, null, rateLimitConfig.ip, null); + } + } + } + } + } + } + + private void something4(List resourceConfigs, String app, String ip, String service) { + String r = ResourceRateLimitConfig.buildResourceId(app, ip, null, service, null); + ResourceConfig rc = new ResourceConfig(r, 0, 0); + resourceConfigs.add(rc); + } } diff --git a/fizz-plugin/pom.xml b/fizz-plugin/pom.xml index 865da1d..4749c02 100644 --- a/fizz-plugin/pom.xml +++ b/fizz-plugin/pom.xml @@ -5,7 +5,7 @@ fizz-gateway-community com.fizzgate - 2.2.0-beta2 + 2.2.0-beta3 ../pom.xml 4.0.0 diff --git a/fizz-spring-boot-starter/pom.xml b/fizz-spring-boot-starter/pom.xml index f2091be..de1eea1 100644 --- a/fizz-spring-boot-starter/pom.xml +++ b/fizz-spring-boot-starter/pom.xml @@ -5,7 +5,7 @@ fizz-gateway-community com.fizzgate - 2.2.0-beta2 + 2.2.0-beta3 ../pom.xml 4.0.0 diff --git a/pom.xml b/pom.xml index 95ceae9..5270cb8 100644 --- a/pom.xml +++ b/pom.xml @@ -31,7 +31,7 @@ fizz-gateway-community ${project.artifactId} fizz gateway community - 2.2.0-beta2 + 2.2.0-beta3 pom fizz-common From 6c013f29bcf49d01912e5f7898b0274382172893 Mon Sep 17 00:00:00 2001 From: hongqiaowei Date: Fri, 25 Jun 2021 17:57:02 +0800 Subject: [PATCH 12/25] Release v2.2.0-beta4 --- fizz-bootstrap/pom.xml | 2 +- fizz-common/pom.xml | 2 +- fizz-core/pom.xml | 2 +- .../ResourceRateLimitConfigService.java | 38 +++++++++++++++---- fizz-plugin/pom.xml | 2 +- fizz-spring-boot-starter/pom.xml | 2 +- pom.xml | 2 +- 7 files changed, 36 insertions(+), 14 deletions(-) diff --git a/fizz-bootstrap/pom.xml b/fizz-bootstrap/pom.xml index 113dac1..1fdddd3 100644 --- a/fizz-bootstrap/pom.xml +++ b/fizz-bootstrap/pom.xml @@ -12,7 +12,7 @@ com.fizzgate fizz-bootstrap - 2.2.0-beta3 + 2.2.0-beta4 1.8 diff --git a/fizz-common/pom.xml b/fizz-common/pom.xml index c55038a..0f3b5cf 100644 --- a/fizz-common/pom.xml +++ b/fizz-common/pom.xml @@ -5,7 +5,7 @@ fizz-gateway-community com.fizzgate - 2.2.0-beta3 + 2.2.0-beta4 ../pom.xml 4.0.0 diff --git a/fizz-core/pom.xml b/fizz-core/pom.xml index 936b83d..8d6e0d9 100644 --- a/fizz-core/pom.xml +++ b/fizz-core/pom.xml @@ -5,7 +5,7 @@ fizz-gateway-community com.fizzgate - 2.2.0-beta3 + 2.2.0-beta4 ../pom.xml 4.0.0 diff --git a/fizz-core/src/main/java/we/stats/ratelimit/ResourceRateLimitConfigService.java b/fizz-core/src/main/java/we/stats/ratelimit/ResourceRateLimitConfigService.java index 6f046da..60203f9 100644 --- a/fizz-core/src/main/java/we/stats/ratelimit/ResourceRateLimitConfigService.java +++ b/fizz-core/src/main/java/we/stats/ratelimit/ResourceRateLimitConfigService.java @@ -181,13 +181,28 @@ public class ResourceRateLimitConfigService { return resourceRateLimitConfigMap; } + // _global, service, app, ip, ip+service public void getParentsTo(String resource, List parentList) { String app = null, ip = null, node = null, service = null, path = null; ResourceRateLimitConfig c = resourceRateLimitConfigMap.get(resource); if (c == null) { - service = ResourceRateLimitConfig.getService(resource); - if (service != null) { - parentList.add(ResourceRateLimitConfig.NODE_RESOURCE); + node = ResourceRateLimitConfig.getNode(resource); + if (node != null && node.equals(ResourceRateLimitConfig.NODE)) { + } else { + service = ResourceRateLimitConfig.getService(resource); + app = ResourceRateLimitConfig.getApp(resource); + ip = ResourceRateLimitConfig.getIp(resource); + if (service == null) { + parentList.add(ResourceRateLimitConfig.NODE_RESOURCE); + } else { + if (ip == null) { + parentList.add(ResourceRateLimitConfig.NODE_RESOURCE); + } else { + String r = ResourceRateLimitConfig.buildResourceId(null, ip, null, null, null); + parentList.add(r); + parentList.add(ResourceRateLimitConfig.NODE_RESOURCE); + } + } } return; } else { @@ -211,7 +226,8 @@ public class ResourceRateLimitConfigService { ResourceRateLimitConfig.buildResourceIdTo(b, app, null, null, service, null); checkRateLimitConfigAndAddTo(b, parentList); ResourceRateLimitConfig.buildResourceIdTo(b, app, null, null, null, null); - checkRateLimitConfigAndAddTo(b, parentList); + // checkRateLimitConfigAndAddTo(b, parentList); + to(parentList, b); } else if (service != null) { ResourceRateLimitConfig.buildResourceIdTo(b, app, null, null, null, null); checkRateLimitConfigAndAddTo(b, parentList); @@ -221,9 +237,11 @@ public class ResourceRateLimitConfigService { if (ip != null) { if (path != null) { ResourceRateLimitConfig.buildResourceIdTo(b, null, ip, null, service, null); - checkRateLimitConfigAndAddTo(b, parentList); + // checkRateLimitConfigAndAddTo(b, parentList); + to(parentList, b); ResourceRateLimitConfig.buildResourceIdTo(b, null, ip, null, null, null); - checkRateLimitConfigAndAddTo(b, parentList); + // checkRateLimitConfigAndAddTo(b, parentList); + to(parentList, b); } else if (service != null) { ResourceRateLimitConfig.buildResourceIdTo(b, null, ip, null, null, null); checkRateLimitConfigAndAddTo(b, parentList); @@ -232,13 +250,17 @@ public class ResourceRateLimitConfigService { if (path != null) { ResourceRateLimitConfig.buildResourceIdTo(b, null, null, null, service, null); - parentList.add(b.toString()); - b.delete(0, b.length()); + to(parentList, b); } parentList.add(ResourceRateLimitConfig.NODE_RESOURCE); } + private void to(List parentList, StringBuilder b) { + parentList.add(b.toString()); + b.delete(0, b.length()); + } + private void checkRateLimitConfigAndAddTo(StringBuilder resourceStringBuilder, List resourceList) { String r = resourceStringBuilder.toString(); ResourceRateLimitConfig c = resourceRateLimitConfigMap.get(r); diff --git a/fizz-plugin/pom.xml b/fizz-plugin/pom.xml index 4749c02..6ee10a4 100644 --- a/fizz-plugin/pom.xml +++ b/fizz-plugin/pom.xml @@ -5,7 +5,7 @@ fizz-gateway-community com.fizzgate - 2.2.0-beta3 + 2.2.0-beta4 ../pom.xml 4.0.0 diff --git a/fizz-spring-boot-starter/pom.xml b/fizz-spring-boot-starter/pom.xml index de1eea1..3045428 100644 --- a/fizz-spring-boot-starter/pom.xml +++ b/fizz-spring-boot-starter/pom.xml @@ -5,7 +5,7 @@ fizz-gateway-community com.fizzgate - 2.2.0-beta3 + 2.2.0-beta4 ../pom.xml 4.0.0 diff --git a/pom.xml b/pom.xml index 5270cb8..8f2926e 100644 --- a/pom.xml +++ b/pom.xml @@ -31,7 +31,7 @@ fizz-gateway-community ${project.artifactId} fizz gateway community - 2.2.0-beta3 + 2.2.0-beta4 pom fizz-common From 7bf668c5c241df43a7d5d4e90f30693a41540a15 Mon Sep 17 00:00:00 2001 From: Francis Dong Date: Mon, 28 Jun 2021 09:36:03 +0800 Subject: [PATCH 13/25] update log default config --- fizz-bootstrap/sh/boot.sh | 2 +- fizz-bootstrap/src/main/resources/log4j2-spring.xml | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/fizz-bootstrap/sh/boot.sh b/fizz-bootstrap/sh/boot.sh index 54e3f71..d719d17 100644 --- a/fizz-bootstrap/sh/boot.sh +++ b/fizz-bootstrap/sh/boot.sh @@ -103,7 +103,7 @@ start() { else echo "starting $APP_NAME ..." rm -f ${PID_FILE} - ${JAVA_CMD} -jar ${JAVA_OPTS} -Dspring.profiles.active=$SPRING_PROFILES_ACTIVE -Denv=$APOLLO_ENV -Dapollo.meta=${APOLLO_META_SERVER} ${APP_DEP_DIR}/${APP_NAME} > ${APP_LOG_DIR}/${APP_NAME}.log 2>&1 & + ${JAVA_CMD} -jar ${JAVA_OPTS} -Dlogging.config=${APP_DEP_DIR}/log4j2-spring.xml -Dspring.profiles.active=$SPRING_PROFILES_ACTIVE -Denv=$APOLLO_ENV -Dapollo.meta=${APOLLO_META_SERVER} ${APP_DEP_DIR}/${APP_NAME} > ${APP_LOG_DIR}/${APP_NAME}.log 2>&1 & echo $! > ${PID_FILE} fi } diff --git a/fizz-bootstrap/src/main/resources/log4j2-spring.xml b/fizz-bootstrap/src/main/resources/log4j2-spring.xml index b5358b9..8429c18 100644 --- a/fizz-bootstrap/src/main/resources/log4j2-spring.xml +++ b/fizz-bootstrap/src/main/resources/log4j2-spring.xml @@ -19,6 +19,7 @@ - + + From abd7e11d593dad028192a93b448a177330b70bb8 Mon Sep 17 00:00:00 2001 From: hongqiaowei Date: Mon, 28 Jun 2021 17:01:56 +0800 Subject: [PATCH 14/25] Release v2.2.0-beta5 --- fizz-bootstrap/pom.xml | 2 +- fizz-common/pom.xml | 2 +- .../src/main/java/we/util/Constants.java | 1 + fizz-core/pom.xml | 2 +- .../java/we/filter/FlowControlFilter.java | 6 ++-- .../ratelimit/ResourceRateLimitConfig.java | 33 +++++++++++-------- .../ResourceRateLimitConfigServiceTests.java | 2 +- .../ResourceRateLimitConfigTests.java | 10 +++--- fizz-plugin/pom.xml | 2 +- fizz-spring-boot-starter/pom.xml | 2 +- pom.xml | 2 +- 11 files changed, 36 insertions(+), 28 deletions(-) diff --git a/fizz-bootstrap/pom.xml b/fizz-bootstrap/pom.xml index 1fdddd3..8cc05e1 100644 --- a/fizz-bootstrap/pom.xml +++ b/fizz-bootstrap/pom.xml @@ -12,7 +12,7 @@ com.fizzgate fizz-bootstrap - 2.2.0-beta4 + 2.2.0-beta5 1.8 diff --git a/fizz-common/pom.xml b/fizz-common/pom.xml index 0f3b5cf..cba3f59 100644 --- a/fizz-common/pom.xml +++ b/fizz-common/pom.xml @@ -5,7 +5,7 @@ fizz-gateway-community com.fizzgate - 2.2.0-beta4 + 2.2.0-beta5 ../pom.xml 4.0.0 diff --git a/fizz-common/src/main/java/we/util/Constants.java b/fizz-common/src/main/java/we/util/Constants.java index eba2a52..398e358 100644 --- a/fizz-common/src/main/java/we/util/Constants.java +++ b/fizz-common/src/main/java/we/util/Constants.java @@ -52,6 +52,7 @@ public final class Constants { public static final char RIGHT_SQUARE_BRACKET = ']'; public static final char LEFT_BRACE = '{'; public static final char RIGHT_BRACE = '}'; + public static final char SQUARE = '^'; public static final char LF = '\n'; public static final char TAB = '\t'; diff --git a/fizz-core/pom.xml b/fizz-core/pom.xml index 8d6e0d9..58fc7a7 100644 --- a/fizz-core/pom.xml +++ b/fizz-core/pom.xml @@ -5,7 +5,7 @@ fizz-gateway-community com.fizzgate - 2.2.0-beta4 + 2.2.0-beta5 ../pom.xml 4.0.0 diff --git a/fizz-core/src/main/java/we/filter/FlowControlFilter.java b/fizz-core/src/main/java/we/filter/FlowControlFilter.java index 6cecf7a..90481e1 100644 --- a/fizz-core/src/main/java/we/filter/FlowControlFilter.java +++ b/fizz-core/src/main/java/we/filter/FlowControlFilter.java @@ -165,7 +165,7 @@ public class FlowControlFilter extends FizzWebFilter { private List getFlowControlConfigs(String app, String ip, String node, String service, String path) { if (log.isDebugEnabled()) { - log.debug("get flow control config by app={}, ip={}, node={}, service={}, path={}", app, ip, node, service, path); + log.debug("get flow control configs by app={}, ip={}, node={}, service={}, path={}", app, ip, node, service, path); } List resourceConfigs = new ArrayList<>(9); StringBuilder b = ThreadContext.getStringBuilder(); @@ -240,7 +240,7 @@ public class FlowControlFilter extends FizzWebFilter { prev = resourceConfigs.get(sz - 1).getResourceId(); prevPrev = resourceConfigs.get(sz - 2).getResourceId(); - if (rateLimitConfig.type == ResourceRateLimitConfig.Type.APP) { + if (rateLimitConfig.type == ResourceRateLimitConfig.Type.APP && rateLimitConfig.path != null) { String app = ResourceRateLimitConfig.getApp(prev); if (app == null) { something4(resourceConfigs, rateLimitConfig.app, null, null); @@ -255,7 +255,7 @@ public class FlowControlFilter extends FizzWebFilter { } } - } else if (rateLimitConfig.type == ResourceRateLimitConfig.Type.IP) { + } else if (rateLimitConfig.type == ResourceRateLimitConfig.Type.IP && rateLimitConfig.path != null) { String ip = ResourceRateLimitConfig.getIp(prev); if (ip == null) { diff --git a/fizz-core/src/main/java/we/stats/ratelimit/ResourceRateLimitConfig.java b/fizz-core/src/main/java/we/stats/ratelimit/ResourceRateLimitConfig.java index 58a499c..039b6e2 100644 --- a/fizz-core/src/main/java/we/stats/ratelimit/ResourceRateLimitConfig.java +++ b/fizz-core/src/main/java/we/stats/ratelimit/ResourceRateLimitConfig.java @@ -18,6 +18,7 @@ package we.stats.ratelimit; import com.fasterxml.jackson.annotation.JsonIgnore; +import org.apache.commons.lang3.StringUtils; import we.util.Constants; import we.util.JacksonUtils; import we.util.Utils; @@ -114,16 +115,22 @@ public class ResourceRateLimitConfig { } } + public void setPath(String p) { + if (StringUtils.isNotBlank(p)) { + path = p; + } + } + private String resourceId = null; @JsonIgnore public String getResourceId() { if (resourceId == null) { resourceId = - (app == null ? "" : app) + '@' + - (ip == null ? "" : ip) + '@' + - (node == null ? "" : node) + '@' + - (service == null ? "" : service) + '@' + + (app == null ? "" : app) + '^' + + (ip == null ? "" : ip) + '^' + + (node == null ? "" : node) + '^' + + (service == null ? "" : service) + '^' + (path == null ? "" : path) ; } @@ -137,15 +144,15 @@ public class ResourceRateLimitConfig { } public static void buildResourceIdTo(StringBuilder b, String app, String ip, String node, String service, String path) { - b.append(app == null ? Constants.Symbol.EMPTY : app) .append(Constants.Symbol.AT); - b.append(ip == null ? Constants.Symbol.EMPTY : ip) .append(Constants.Symbol.AT); - b.append(node == null ? Constants.Symbol.EMPTY : node) .append(Constants.Symbol.AT); - b.append(service == null ? Constants.Symbol.EMPTY : service) .append(Constants.Symbol.AT); + b.append(app == null ? Constants.Symbol.EMPTY : app) .append(Constants.Symbol.SQUARE); + b.append(ip == null ? Constants.Symbol.EMPTY : ip) .append(Constants.Symbol.SQUARE); + b.append(node == null ? Constants.Symbol.EMPTY : node) .append(Constants.Symbol.SQUARE); + b.append(service == null ? Constants.Symbol.EMPTY : service) .append(Constants.Symbol.SQUARE); b.append(path == null ? Constants.Symbol.EMPTY : path); } public static String getApp(String resource) { - int i = resource.indexOf(Constants.Symbol.AT); + int i = resource.indexOf(Constants.Symbol.SQUARE); if (i == 0) { return null; } else { @@ -154,7 +161,7 @@ public class ResourceRateLimitConfig { } public static String getIp(String resource) { - String extract = Utils.extract(resource, Constants.Symbol.AT, 1); + String extract = Utils.extract(resource, Constants.Symbol.SQUARE, 1); if (extract.equals(Constants.Symbol.EMPTY)) { return null; } @@ -162,7 +169,7 @@ public class ResourceRateLimitConfig { } public static String getNode(String resource) { - String extract = Utils.extract(resource, Constants.Symbol.AT, 2); + String extract = Utils.extract(resource, Constants.Symbol.SQUARE, 2); if (extract.equals(Constants.Symbol.EMPTY)) { return null; } @@ -170,7 +177,7 @@ public class ResourceRateLimitConfig { } public static String getService(String resource) { - String extract = Utils.extract(resource, Constants.Symbol.AT, 3); + String extract = Utils.extract(resource, Constants.Symbol.SQUARE, 3); if (extract.equals(Constants.Symbol.EMPTY)) { return null; } @@ -178,7 +185,7 @@ public class ResourceRateLimitConfig { } public static String getPath(String resource) { - int i = resource.lastIndexOf(Constants.Symbol.AT); + int i = resource.lastIndexOf(Constants.Symbol.SQUARE); if (i == resource.length() - 1) { return null; } else { diff --git a/fizz-core/src/test/java/we/stats/ratelimit/ResourceRateLimitConfigServiceTests.java b/fizz-core/src/test/java/we/stats/ratelimit/ResourceRateLimitConfigServiceTests.java index 8c2bce0..ae758bd 100644 --- a/fizz-core/src/test/java/we/stats/ratelimit/ResourceRateLimitConfigServiceTests.java +++ b/fizz-core/src/test/java/we/stats/ratelimit/ResourceRateLimitConfigServiceTests.java @@ -57,7 +57,7 @@ public class ResourceRateLimitConfigServiceTests { assertEquals(resourceRateLimitConfig.concurrents, 66); // System.err.println(resourceRateLimitConfig); // Thread.currentThread().join(); - resourceRateLimitConfig = resourceRateLimitConfigService.getResourceRateLimitConfig("xapp@@@yservice@"); + resourceRateLimitConfig = resourceRateLimitConfigService.getResourceRateLimitConfig("xapp^^^yservice^"); assertEquals(resourceRateLimitConfig.concurrents, 88); Thread.sleep(4000); diff --git a/fizz-core/src/test/java/we/stats/ratelimit/ResourceRateLimitConfigTests.java b/fizz-core/src/test/java/we/stats/ratelimit/ResourceRateLimitConfigTests.java index 80aaac2..2a2fab0 100644 --- a/fizz-core/src/test/java/we/stats/ratelimit/ResourceRateLimitConfigTests.java +++ b/fizz-core/src/test/java/we/stats/ratelimit/ResourceRateLimitConfigTests.java @@ -23,7 +23,7 @@ public class ResourceRateLimitConfigTests { String resourceRateLimitConfigJson = "{\"concurrents\":1000,\"enable\":1,\"id\":1,\"isDeleted\":0,\"qps\":500, \"type\":1, \"resource\":\"_global\" }"; ResourceRateLimitConfig c = JacksonUtils.readValue(resourceRateLimitConfigJson, ResourceRateLimitConfig.class); String resourceId = c.getResourceId(); - assertEquals("@@_global@@", resourceId); + assertEquals("^^_global^^", resourceId); String node = ResourceRateLimitConfig.getNode(resourceId); assertEquals("_global", node); @@ -31,17 +31,17 @@ public class ResourceRateLimitConfigTests { resourceRateLimitConfigJson = "{\"concurrents\":1000,\"enable\":1,\"id\":1,\"isDeleted\":0,\"qps\":500, \"type\":2, \"resource\":\"service_default\" }"; c = JacksonUtils.readValue(resourceRateLimitConfigJson, ResourceRateLimitConfig.class); resourceId = c.getResourceId(); - assertEquals("@@@service_default@", resourceId); + assertEquals("^^^service_default^", resourceId); resourceRateLimitConfigJson = "{\"concurrents\":1000,\"enable\":1,\"id\":1,\"isDeleted\":0,\"qps\":500, \"type\":3, \"resource\":\"xservice\" }"; c = JacksonUtils.readValue(resourceRateLimitConfigJson, ResourceRateLimitConfig.class); resourceId = c.getResourceId(); - assertEquals("@@@xservice@", resourceId); + assertEquals("^^^xservice^", resourceId); resourceId = ResourceRateLimitConfig.buildResourceId(null, null, ResourceRateLimitConfig.NODE, null, null); - assertEquals("@@_global@@", resourceId); + assertEquals("^^_global^^", resourceId); resourceId = ResourceRateLimitConfig.buildResourceId(null, "192.168.1.1", null, "xservice", null); - assertEquals("@192.168.1.1@@xservice@", resourceId); + assertEquals("^192.168.1.1^^xservice^", resourceId); } } diff --git a/fizz-plugin/pom.xml b/fizz-plugin/pom.xml index 6ee10a4..8591649 100644 --- a/fizz-plugin/pom.xml +++ b/fizz-plugin/pom.xml @@ -5,7 +5,7 @@ fizz-gateway-community com.fizzgate - 2.2.0-beta4 + 2.2.0-beta5 ../pom.xml 4.0.0 diff --git a/fizz-spring-boot-starter/pom.xml b/fizz-spring-boot-starter/pom.xml index 3045428..f9860b0 100644 --- a/fizz-spring-boot-starter/pom.xml +++ b/fizz-spring-boot-starter/pom.xml @@ -5,7 +5,7 @@ fizz-gateway-community com.fizzgate - 2.2.0-beta4 + 2.2.0-beta5 ../pom.xml 4.0.0 diff --git a/pom.xml b/pom.xml index 8f2926e..10f503a 100644 --- a/pom.xml +++ b/pom.xml @@ -31,7 +31,7 @@ fizz-gateway-community ${project.artifactId} fizz gateway community - 2.2.0-beta4 + 2.2.0-beta5 pom fizz-common From 54bc0c23b96c855293f2ba3734138fd86368b69f Mon Sep 17 00:00:00 2001 From: hongqiaowei Date: Wed, 30 Jun 2021 14:47:33 +0800 Subject: [PATCH 15/25] Release v2.2.0-beta6 --- fizz-bootstrap/pom.xml | 2 +- fizz-common/pom.xml | 2 +- fizz-core/pom.xml | 2 +- .../java/we/config/FlowStatSchedConfig.java | 3 +- .../java/we/filter/FlowControlFilter.java | 36 +++++++++++-------- .../ratelimit/ResourceRateLimitConfig.java | 14 ++++++-- fizz-plugin/pom.xml | 2 +- fizz-spring-boot-starter/pom.xml | 2 +- pom.xml | 2 +- 9 files changed, 41 insertions(+), 24 deletions(-) diff --git a/fizz-bootstrap/pom.xml b/fizz-bootstrap/pom.xml index 8cc05e1..628a4c9 100644 --- a/fizz-bootstrap/pom.xml +++ b/fizz-bootstrap/pom.xml @@ -12,7 +12,7 @@ com.fizzgate fizz-bootstrap - 2.2.0-beta5 + 2.2.0-beta6 1.8 diff --git a/fizz-common/pom.xml b/fizz-common/pom.xml index cba3f59..71294c2 100644 --- a/fizz-common/pom.xml +++ b/fizz-common/pom.xml @@ -5,7 +5,7 @@ fizz-gateway-community com.fizzgate - 2.2.0-beta5 + 2.2.0-beta6 ../pom.xml 4.0.0 diff --git a/fizz-core/pom.xml b/fizz-core/pom.xml index 58fc7a7..317819f 100644 --- a/fizz-core/pom.xml +++ b/fizz-core/pom.xml @@ -5,7 +5,7 @@ fizz-gateway-community com.fizzgate - 2.2.0-beta5 + 2.2.0-beta6 ../pom.xml 4.0.0 diff --git a/fizz-core/src/main/java/we/config/FlowStatSchedConfig.java b/fizz-core/src/main/java/we/config/FlowStatSchedConfig.java index ec820bc..cf8f6c4 100644 --- a/fizz-core/src/main/java/we/config/FlowStatSchedConfig.java +++ b/fizz-core/src/main/java/we/config/FlowStatSchedConfig.java @@ -240,7 +240,8 @@ public class FlowStatSchedConfig extends SchedConfig { rt.convertAndSend(flowStatSchedConfigProperties.getQueue(), msg).subscribe(); } if (log.isDebugEnabled()) { - log.debug("report " + toDP19(timeWin) + " win10: " + msg); + String wt = 'w' + toDP19(timeWin); + log.debug("report " + wt + ": " + msg, LogService.BIZ_ID, wt); } } } diff --git a/fizz-core/src/main/java/we/filter/FlowControlFilter.java b/fizz-core/src/main/java/we/filter/FlowControlFilter.java index 90481e1..fcda08f 100644 --- a/fizz-core/src/main/java/we/filter/FlowControlFilter.java +++ b/fizz-core/src/main/java/we/filter/FlowControlFilter.java @@ -255,23 +255,31 @@ public class FlowControlFilter extends FizzWebFilter { } } - } else if (rateLimitConfig.type == ResourceRateLimitConfig.Type.IP && rateLimitConfig.path != null) { + } else if (rateLimitConfig.type == ResourceRateLimitConfig.Type.IP) { - String ip = ResourceRateLimitConfig.getIp(prev); - if (ip == null) { - something4(resourceConfigs, null, rateLimitConfig.ip, null); - something4(resourceConfigs, null, rateLimitConfig.ip, rateLimitConfig.service); - } else { - String service = ResourceRateLimitConfig.getService(prev); - if (service == null) { - something4(resourceConfigs, null, rateLimitConfig.ip, rateLimitConfig.service); + if (rateLimitConfig.service == null && rateLimitConfig.path == null) { + } else if (rateLimitConfig.path == null) { + String ip = ResourceRateLimitConfig.getIp(prev); + if (ip == null) { + something4(resourceConfigs, null, rateLimitConfig.ip, null); + } } else { - ip = ResourceRateLimitConfig.getIp(prevPrev); - if (ip == null) { - something4(resourceConfigs, null, rateLimitConfig.ip, null); - } + String ip = ResourceRateLimitConfig.getIp(prev); + if (ip == null) { + something4(resourceConfigs, null, rateLimitConfig.ip, null); + something4(resourceConfigs, null, rateLimitConfig.ip, rateLimitConfig.service); + } else { + String service = ResourceRateLimitConfig.getService(prev); + if (service == null) { + something4(resourceConfigs, null, rateLimitConfig.ip, rateLimitConfig.service); + } else { + ip = ResourceRateLimitConfig.getIp(prevPrev); + if (ip == null) { + something4(resourceConfigs, null, rateLimitConfig.ip, null); + } + } + } } - } } } } diff --git a/fizz-core/src/main/java/we/stats/ratelimit/ResourceRateLimitConfig.java b/fizz-core/src/main/java/we/stats/ratelimit/ResourceRateLimitConfig.java index 039b6e2..464f065 100644 --- a/fizz-core/src/main/java/we/stats/ratelimit/ResourceRateLimitConfig.java +++ b/fizz-core/src/main/java/we/stats/ratelimit/ResourceRateLimitConfig.java @@ -98,9 +98,11 @@ public class ResourceRateLimitConfig { } public void setResource(String r) { - resource = r; - if (!resource.equals(NODE)) { - service = resource; + if (StringUtils.isNotBlank(r)) { + resource = r; + if (!resource.equals(NODE)) { + service = resource; + } } } @@ -115,6 +117,12 @@ public class ResourceRateLimitConfig { } } + public void setService(String s) { + if (StringUtils.isNotBlank(s)) { + service = s; + } + } + public void setPath(String p) { if (StringUtils.isNotBlank(p)) { path = p; diff --git a/fizz-plugin/pom.xml b/fizz-plugin/pom.xml index 8591649..6079cba 100644 --- a/fizz-plugin/pom.xml +++ b/fizz-plugin/pom.xml @@ -5,7 +5,7 @@ fizz-gateway-community com.fizzgate - 2.2.0-beta5 + 2.2.0-beta6 ../pom.xml 4.0.0 diff --git a/fizz-spring-boot-starter/pom.xml b/fizz-spring-boot-starter/pom.xml index f9860b0..37a19b0 100644 --- a/fizz-spring-boot-starter/pom.xml +++ b/fizz-spring-boot-starter/pom.xml @@ -5,7 +5,7 @@ fizz-gateway-community com.fizzgate - 2.2.0-beta5 + 2.2.0-beta6 ../pom.xml 4.0.0 diff --git a/pom.xml b/pom.xml index 10f503a..7f8c28b 100644 --- a/pom.xml +++ b/pom.xml @@ -31,7 +31,7 @@ fizz-gateway-community ${project.artifactId} fizz gateway community - 2.2.0-beta5 + 2.2.0-beta6 pom fizz-common From d343fcfe8e0921f8de1eb6b3c79c9cc888eb831e Mon Sep 17 00:00:00 2001 From: Francis Dong Date: Mon, 5 Jul 2021 17:02:27 +0800 Subject: [PATCH 16/25] support total block requests --- .../src/main/java/we/stats/FlowStat.java | 36 +++++++++++++++++++ .../src/main/java/we/stats/ResourceStat.java | 11 ++++++ .../src/main/java/we/stats/TimeSlot.java | 13 +++++++ .../main/java/we/stats/TimeWindowStat.java | 13 +++++++ 4 files changed, 73 insertions(+) diff --git a/fizz-core/src/main/java/we/stats/FlowStat.java b/fizz-core/src/main/java/we/stats/FlowStat.java index 417fcf8..6ad7bcb 100644 --- a/fizz-core/src/main/java/we/stats/FlowStat.java +++ b/fizz-core/src/main/java/we/stats/FlowStat.java @@ -28,6 +28,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.function.BiFunction; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -110,6 +111,21 @@ public class FlowStat { * @return IncrRequestResult */ public IncrRequestResult incrRequest(List resourceConfigs, long curTimeSlotId) { + return incrRequest(resourceConfigs, curTimeSlotId, null); + } + + /** + * Increase concurrent request counter for given resources chain + * + * @param resourceConfigs Resource configurations + * @param curTimeSlotId current time slot ID, it should be generated by + * Flowstat.currentTimeSlotId() + * @param totalBlockFunc [optional] callback function for statistic of total + * block requests of the resource and its parent resources + * @return IncrRequestResult + */ + public IncrRequestResult incrRequest(List resourceConfigs, long curTimeSlotId, + BiFunction, List> totalBlockFunc) { if (resourceConfigs == null || resourceConfigs.size() == 0) { return null; } @@ -126,6 +142,16 @@ public class FlowStat { long n = resourceStat.getConcurrentRequests().get(); if (n >= maxCon) { resourceStat.incrBlockRequestToTimeSlot(curTimeSlotId); + if (totalBlockFunc != null) { + List parentResCfgs = totalBlockFunc.apply(resourceConfig, + resourceConfigs); + if (parentResCfgs != null && parentResCfgs.size() > 0) { + for (ResourceConfig pResCfg : parentResCfgs) { + getResourceStat(pResCfg.getResourceId()) + .incrTotalBlockRequestToTimeSlot(curTimeSlotId); + } + } + } return IncrRequestResult.block(resourceConfig.getResourceId(), BlockType.CONCURRENT_REQUEST); } @@ -136,6 +162,16 @@ public class FlowStat { long total = resourceStat.getTimeSlot(curTimeSlotId).getCounter().get(); if (total >= maxQPS) { resourceStat.incrBlockRequestToTimeSlot(curTimeSlotId); + if (totalBlockFunc != null) { + List parentResCfgs = totalBlockFunc.apply(resourceConfig, + resourceConfigs); + if (parentResCfgs != null && parentResCfgs.size() > 0) { + for (ResourceConfig pResCfg : parentResCfgs) { + getResourceStat(pResCfg.getResourceId()) + .incrTotalBlockRequestToTimeSlot(curTimeSlotId); + } + } + } return IncrRequestResult.block(resourceConfig.getResourceId(), BlockType.QPS); } } diff --git a/fizz-core/src/main/java/we/stats/ResourceStat.java b/fizz-core/src/main/java/we/stats/ResourceStat.java index d8a23e6..92b3652 100644 --- a/fizz-core/src/main/java/we/stats/ResourceStat.java +++ b/fizz-core/src/main/java/we/stats/ResourceStat.java @@ -128,6 +128,14 @@ public class ResourceStat { public void incrBlockRequestToTimeSlot(long timeSlotId) { this.getTimeSlot(timeSlotId).getBlockRequests().incrementAndGet(); } + + /** + * Increase total block request to the specified time slot + * + */ + public void incrTotalBlockRequestToTimeSlot(long timeSlotId) { + this.getTimeSlot(timeSlotId).getTotalBlockRequests().incrementAndGet(); + } /** * Add request to the specified time slot @@ -197,6 +205,7 @@ public class ResourceStat { long peakConcurrences = 0; long errors = 0; long blockReqs = 0; + long totalBlockReqs = 0; long compReqs = 0; for (long i = startSlotId; i < endSlotId;) { if (timeSlots.containsKey(i)) { @@ -210,6 +219,7 @@ public class ResourceStat { totalRt = totalRt + timeSlot.getTotalRt().get(); errors = errors + timeSlot.getErrors().get(); blockReqs = blockReqs + timeSlot.getBlockRequests().get(); + totalBlockReqs = totalBlockReqs + timeSlot.getTotalBlockRequests().get(); compReqs = compReqs + timeSlot.getCompReqs().get(); } i = i + FlowStat.INTERVAL; @@ -220,6 +230,7 @@ public class ResourceStat { tws.setTotal(totalReqs); tws.setErrors(errors); tws.setBlockRequests(blockReqs); + tws.setTotalBlockRequests(totalBlockReqs); tws.setCompReqs(compReqs); if (compReqs > 0) { diff --git a/fizz-core/src/main/java/we/stats/TimeSlot.java b/fizz-core/src/main/java/we/stats/TimeSlot.java index 13743cb..765eed4 100644 --- a/fizz-core/src/main/java/we/stats/TimeSlot.java +++ b/fizz-core/src/main/java/we/stats/TimeSlot.java @@ -70,6 +70,11 @@ public class TimeSlot { * Block requests
*/ private AtomicLong blockRequests = new AtomicLong(0); + + /** + * Total block requests of the resource and its underlying resources
+ */ + private AtomicLong totalBlockRequests = new AtomicLong(0); public TimeSlot(long id) { this.id = id; @@ -181,4 +186,12 @@ public class TimeSlot { this.compReqs = compReqs; } + public AtomicLong getTotalBlockRequests() { + return totalBlockRequests; + } + + public void setTotalBlockRequests(AtomicLong totalBlockRequests) { + this.totalBlockRequests = totalBlockRequests; + } + } diff --git a/fizz-core/src/main/java/we/stats/TimeWindowStat.java b/fizz-core/src/main/java/we/stats/TimeWindowStat.java index 796f4e2..2204208 100644 --- a/fizz-core/src/main/java/we/stats/TimeWindowStat.java +++ b/fizz-core/src/main/java/we/stats/TimeWindowStat.java @@ -79,6 +79,11 @@ public class TimeWindowStat { * Block requests */ private Long blockRequests; + + /** + * Total block requests + */ + private Long totalBlockRequests; public Long getBlockRequests() { return blockRequests; @@ -168,4 +173,12 @@ public class TimeWindowStat { this.compReqs = compReqs; } + public Long getTotalBlockRequests() { + return totalBlockRequests; + } + + public void setTotalBlockRequests(Long totalBlockRequests) { + this.totalBlockRequests = totalBlockRequests; + } + } From 95465f6a83d4dcd0a37d049e161b605951f018e0 Mon Sep 17 00:00:00 2001 From: hongqiaowei Date: Tue, 6 Jul 2021 09:00:04 +0800 Subject: [PATCH 17/25] handle total block requests --- .../java/we/config/FlowStatSchedConfig.java | 73 +++++---- .../java/we/filter/FlowControlFilter.java | 49 +++++- .../ResourceRateLimitConfigService.java | 154 +++++++++--------- 3 files changed, 167 insertions(+), 109 deletions(-) diff --git a/fizz-core/src/main/java/we/config/FlowStatSchedConfig.java b/fizz-core/src/main/java/we/config/FlowStatSchedConfig.java index cf8f6c4..a4034af 100644 --- a/fizz-core/src/main/java/we/config/FlowStatSchedConfig.java +++ b/fizz-core/src/main/java/we/config/FlowStatSchedConfig.java @@ -92,7 +92,7 @@ public class FlowStatSchedConfig extends SchedConfig { private long startTimeSlot = 0; - private Map resourceTimeWindow2totalBlockRequestsMap = new HashMap<>(128); + // private Map resourceTimeWindow2totalBlockRequestsMap = new HashMap<>(128); @Scheduled(cron = "${flow-stat-sched.cron}") public void sched() { @@ -112,24 +112,24 @@ public class FlowStatSchedConfig extends SchedConfig { return; } - resourceTimeWindow2totalBlockRequestsMap.clear(); - resourceTimeWindowStats.forEach(rtws -> { - String resource = rtws.getResourceId(); - List wins = rtws.getWindows(); - wins.forEach(w -> { - long t = w.getStartTime(); - long blockRequests = w.getBlockRequests(); - resourceTimeWindow2totalBlockRequestsMap.put(resource + t, new AtomicLong(blockRequests)); - }); - }); + // resourceTimeWindow2totalBlockRequestsMap.clear(); + // resourceTimeWindowStats.forEach(rtws -> { + // String resource = rtws.getResourceId(); + // List wins = rtws.getWindows(); + // wins.forEach(w -> { + // long t = w.getStartTime(); + // long blockRequests = w.getBlockRequests(); + // resourceTimeWindow2totalBlockRequestsMap.put(resource + t, new AtomicLong(blockRequests)); + // }); + // }); - resourceTimeWindowStats.forEach(rtws -> { - String resource = rtws.getResourceId(); - List wins = rtws.getWindows(); - wins.forEach(w -> { - accumulateParents(resource, w.getStartTime(), w.getBlockRequests()); - }); - }); + // resourceTimeWindowStats.forEach(rtws -> { + // String resource = rtws.getResourceId(); + // List wins = rtws.getWindows(); + // wins.forEach(w -> { + // accumulateParents(resource, w.getStartTime(), w.getBlockRequests()); + // }); + // }); resourceTimeWindowStats.forEach( rtws -> { @@ -138,7 +138,7 @@ public class FlowStatSchedConfig extends SchedConfig { int type = ResourceRateLimitConfig.Type.NODE, id = 0; ResourceRateLimitConfig c = resourceRateLimitConfigService.getResourceRateLimitConfig(resource); - if (c == null) { // _global, service, app, ip, ip+service + if (c == null) { // _global, service, app, app+service, ip, ip+service node = ResourceRateLimitConfig.getNode(resource); if (node != null && node.equals(ResourceRateLimitConfig.NODE)) { } else { @@ -157,10 +157,14 @@ public class FlowStatSchedConfig extends SchedConfig { } } } else { - if (pi == null) { + if (app == null && pi == null) { type = ResourceRateLimitConfig.Type.SERVICE_DEFAULT; } else { - type = ResourceRateLimitConfig.Type.IP; + if (app == null) { + type = ResourceRateLimitConfig.Type.IP; + } else { + type = ResourceRateLimitConfig.Type.APP; + } } } } @@ -186,8 +190,9 @@ public class FlowStatSchedConfig extends SchedConfig { qps = rps.doubleValue(); } - AtomicLong totalBlockRequests = resourceTimeWindow2totalBlockRequestsMap.get(resource + timeWin); - long tbrs = (totalBlockRequests == null ? w.getBlockRequests() : totalBlockRequests.longValue()); + // AtomicLong totalBlockRequests = resourceTimeWindow2totalBlockRequestsMap.get(resource + timeWin); + // long tbrs = (totalBlockRequests == null ? w.getBlockRequests() : w.getBlockRequests() + totalBlockRequests.longValue()); + long tbrs = w.getTotalBlockRequests(); b.append(Constants.Symbol.LEFT_BRACE); b.append(_ip); toJsonStringValue(b, ip); b.append(Constants.Symbol.COMMA); @@ -253,17 +258,17 @@ public class FlowStatSchedConfig extends SchedConfig { } } - private void accumulateParents(String resource, long timeWin, long blockRequests) { - List prl = ThreadContext.getArrayList(parentResourceList, String.class); - resourceRateLimitConfigService.getParentsTo(resource, prl); - for (int i = 0; i < prl.size(); i++) { - String parentResource = prl.get(i); - AtomicLong parentTotalBlockRequests = resourceTimeWindow2totalBlockRequestsMap.get(parentResource + timeWin); - if (parentTotalBlockRequests != null) { - parentTotalBlockRequests.addAndGet(blockRequests); - } - } - } + // private void accumulateParents(String resource, long timeWin, long blockRequests) { + // List prl = ThreadContext.getArrayList(parentResourceList, String.class); + // resourceRateLimitConfigService.getParentsTo(resource, prl); + // for (int i = 0; i < prl.size(); i++) { + // String parentResource = prl.get(i); + // AtomicLong parentTotalBlockRequests = resourceTimeWindow2totalBlockRequestsMap.get(parentResource + timeWin); + // if (parentTotalBlockRequests != null) { + // parentTotalBlockRequests.addAndGet(blockRequests); + // } + // } + // } private long getRecentEndTimeSlot(FlowStat flowStat) { long currentTimeSlot = flowStat.currentTimeSlotId(); diff --git a/fizz-core/src/main/java/we/filter/FlowControlFilter.java b/fizz-core/src/main/java/we/filter/FlowControlFilter.java index fcda08f..af2001c 100644 --- a/fizz-core/src/main/java/we/filter/FlowControlFilter.java +++ b/fizz-core/src/main/java/we/filter/FlowControlFilter.java @@ -120,7 +120,9 @@ public class FlowControlFilter extends FizzWebFilter { long currentTimeSlot = flowStat.currentTimeSlotId(); List resourceConfigs = getFlowControlConfigs(app, ip, null, service, path); - IncrRequestResult result = flowStat.incrRequest(resourceConfigs, currentTimeSlot); + IncrRequestResult result = flowStat.incrRequest(resourceConfigs, currentTimeSlot, (rc, rcs) -> { + return getResourceConfigItselfAndParents(rc, rcs); + }); if (result != null && !result.isSuccess()) { String blockedResourceId = result.getBlockedResourceId(); @@ -163,6 +165,49 @@ public class FlowControlFilter extends FizzWebFilter { return chain.filter(exchange); } + private List getResourceConfigItselfAndParents(ResourceConfig rc, List rcs) { + boolean check = false; + String rcId = rc.getResourceId(); + String rcApp = ResourceRateLimitConfig.getApp(rcId); + String rcIp = ResourceRateLimitConfig.getIp(rcId); + List result = new ArrayList<>(); + for (int i = rcs.size() - 1; i > -1; i--) { + ResourceConfig r = rcs.get(i); + String id = r.getResourceId(); + String app = ResourceRateLimitConfig.getApp(id); + String ip = ResourceRateLimitConfig.getIp(id); + String path = ResourceRateLimitConfig.getPath(id); + if (check) { + if (rcIp != null) { + if (ip != null) { + result.add(r); + } else { + if (app == null && path == null) { + result.add(r); + } + } + } else if (rcApp != null) { + if (app != null) { + result.add(r); + } else { + if (path == null) { + result.add(r); + } + } + } else { + result.add(r); + } + } else if (id.equals(rcId)) { + result.add(r); + check = true; + } + } + if (log.isDebugEnabled()) { + log.debug("getResourceConfigItselfAndParents:\n" + JacksonUtils.writeValueAsString(rc) + '\n' + JacksonUtils.writeValueAsString(result)); + } + return result; + } + private List getFlowControlConfigs(String app, String ip, String node, String service, String path) { if (log.isDebugEnabled()) { log.debug("get flow control configs by app={}, ip={}, node={}, service={}, path={}", app, ip, node, service, path); @@ -244,9 +289,11 @@ public class FlowControlFilter extends FizzWebFilter { String app = ResourceRateLimitConfig.getApp(prev); if (app == null) { something4(resourceConfigs, rateLimitConfig.app, null, null); + something4(resourceConfigs, rateLimitConfig.app, null, rateLimitConfig.service); } else { String service = ResourceRateLimitConfig.getService(prev); if (service == null) { + something4(resourceConfigs, rateLimitConfig.app, null, rateLimitConfig.service); } else { app = ResourceRateLimitConfig.getApp(prevPrev); if (app == null) { diff --git a/fizz-core/src/main/java/we/stats/ratelimit/ResourceRateLimitConfigService.java b/fizz-core/src/main/java/we/stats/ratelimit/ResourceRateLimitConfigService.java index 60203f9..9cb0b08 100644 --- a/fizz-core/src/main/java/we/stats/ratelimit/ResourceRateLimitConfigService.java +++ b/fizz-core/src/main/java/we/stats/ratelimit/ResourceRateLimitConfigService.java @@ -181,80 +181,86 @@ public class ResourceRateLimitConfigService { return resourceRateLimitConfigMap; } - // _global, service, app, ip, ip+service - public void getParentsTo(String resource, List parentList) { - String app = null, ip = null, node = null, service = null, path = null; - ResourceRateLimitConfig c = resourceRateLimitConfigMap.get(resource); - if (c == null) { - node = ResourceRateLimitConfig.getNode(resource); - if (node != null && node.equals(ResourceRateLimitConfig.NODE)) { - } else { - service = ResourceRateLimitConfig.getService(resource); - app = ResourceRateLimitConfig.getApp(resource); - ip = ResourceRateLimitConfig.getIp(resource); - if (service == null) { - parentList.add(ResourceRateLimitConfig.NODE_RESOURCE); - } else { - if (ip == null) { - parentList.add(ResourceRateLimitConfig.NODE_RESOURCE); - } else { - String r = ResourceRateLimitConfig.buildResourceId(null, ip, null, null, null); - parentList.add(r); - parentList.add(ResourceRateLimitConfig.NODE_RESOURCE); - } - } - } - return; - } else { - if (c.type == ResourceRateLimitConfig.Type.NODE) { - return; - } - if (c.type == ResourceRateLimitConfig.Type.SERVICE) { - parentList.add(ResourceRateLimitConfig.NODE_RESOURCE); - return; - } - app = c.app; - ip = c.ip; - service = c.service; - path = c.path; - } - - StringBuilder b = ThreadContext.getStringBuilder(); - - if (app != null) { - if (path != null) { - ResourceRateLimitConfig.buildResourceIdTo(b, app, null, null, service, null); - checkRateLimitConfigAndAddTo(b, parentList); - ResourceRateLimitConfig.buildResourceIdTo(b, app, null, null, null, null); - // checkRateLimitConfigAndAddTo(b, parentList); - to(parentList, b); - } else if (service != null) { - ResourceRateLimitConfig.buildResourceIdTo(b, app, null, null, null, null); - checkRateLimitConfigAndAddTo(b, parentList); - } - } - - if (ip != null) { - if (path != null) { - ResourceRateLimitConfig.buildResourceIdTo(b, null, ip, null, service, null); - // checkRateLimitConfigAndAddTo(b, parentList); - to(parentList, b); - ResourceRateLimitConfig.buildResourceIdTo(b, null, ip, null, null, null); - // checkRateLimitConfigAndAddTo(b, parentList); - to(parentList, b); - } else if (service != null) { - ResourceRateLimitConfig.buildResourceIdTo(b, null, ip, null, null, null); - checkRateLimitConfigAndAddTo(b, parentList); - } - } - - if (path != null) { - ResourceRateLimitConfig.buildResourceIdTo(b, null, null, null, service, null); - to(parentList, b); - } - - parentList.add(ResourceRateLimitConfig.NODE_RESOURCE); - } + // _global, service, app, app+service, ip, ip+service + // public void getParentsTo(String resource, List parentList) { + // String app = null, ip = null, node = null, service = null, path = null; + // ResourceRateLimitConfig c = resourceRateLimitConfigMap.get(resource); + // if (c == null) { + // node = ResourceRateLimitConfig.getNode(resource); + // if (node != null && node.equals(ResourceRateLimitConfig.NODE)) { + // } else { + // service = ResourceRateLimitConfig.getService(resource); + // app = ResourceRateLimitConfig.getApp(resource); + // ip = ResourceRateLimitConfig.getIp(resource); + // if (service == null) { // or app ip + // parentList.add(ResourceRateLimitConfig.NODE_RESOURCE); + // } else { + // if (app == null && ip == null) { + // parentList.add(ResourceRateLimitConfig.NODE_RESOURCE); + // } else { + // String r = null; + // if (app == null) { + // r = ResourceRateLimitConfig.buildResourceId(null, ip, null, null, null); + // } else { + // r = ResourceRateLimitConfig.buildResourceId(app, null, null, null, null); + // } + // parentList.add(r); + // parentList.add( ResourceRateLimitConfig.buildResourceId(null, null, null, service, null) ); + // parentList.add(ResourceRateLimitConfig.NODE_RESOURCE); + // } + // } + // } + // return; + // } else { + // if (c.type == ResourceRateLimitConfig.Type.NODE) { + // return; + // } + // if (c.type == ResourceRateLimitConfig.Type.SERVICE) { + // parentList.add(ResourceRateLimitConfig.NODE_RESOURCE); + // return; + // } + // app = c.app; + // ip = c.ip; + // service = c.service; + // path = c.path; + // } + // + // StringBuilder b = ThreadContext.getStringBuilder(); + // + // if (app != null) { + // if (path != null) { + // ResourceRateLimitConfig.buildResourceIdTo(b, app, null, null, service, null); + // checkRateLimitConfigAndAddTo(b, parentList); + // ResourceRateLimitConfig.buildResourceIdTo(b, app, null, null, null, null); + // // checkRateLimitConfigAndAddTo(b, parentList); + // to(parentList, b); + // } else if (service != null) { + // ResourceRateLimitConfig.buildResourceIdTo(b, app, null, null, null, null); + // checkRateLimitConfigAndAddTo(b, parentList); + // } + // } + // + // if (ip != null) { + // if (path != null) { + // ResourceRateLimitConfig.buildResourceIdTo(b, null, ip, null, service, null); + // // checkRateLimitConfigAndAddTo(b, parentList); + // to(parentList, b); + // ResourceRateLimitConfig.buildResourceIdTo(b, null, ip, null, null, null); + // // checkRateLimitConfigAndAddTo(b, parentList); + // to(parentList, b); + // } else if (service != null) { + // ResourceRateLimitConfig.buildResourceIdTo(b, null, ip, null, null, null); + // checkRateLimitConfigAndAddTo(b, parentList); + // } + // } + // + // if (path != null) { + // ResourceRateLimitConfig.buildResourceIdTo(b, null, null, null, service, null); + // to(parentList, b); + // } + // + // parentList.add(ResourceRateLimitConfig.NODE_RESOURCE); + // } private void to(List parentList, StringBuilder b) { parentList.add(b.toString()); From 7cde4105fd2a287b62c79db308b58c1b87912bd4 Mon Sep 17 00:00:00 2001 From: hongqiaowei Date: Tue, 6 Jul 2021 17:24:20 +0800 Subject: [PATCH 18/25] Release 2.2.0-beta7 --- fizz-bootstrap/pom.xml | 2 +- fizz-common/pom.xml | 2 +- fizz-core/pom.xml | 2 +- .../java/we/filter/FlowControlFilter.java | 34 +++++++++++-------- fizz-plugin/pom.xml | 2 +- fizz-spring-boot-starter/pom.xml | 2 +- pom.xml | 2 +- 7 files changed, 26 insertions(+), 20 deletions(-) diff --git a/fizz-bootstrap/pom.xml b/fizz-bootstrap/pom.xml index 628a4c9..8b6a76e 100644 --- a/fizz-bootstrap/pom.xml +++ b/fizz-bootstrap/pom.xml @@ -12,7 +12,7 @@ com.fizzgate fizz-bootstrap - 2.2.0-beta6 + 2.2.0-beta7 1.8 diff --git a/fizz-common/pom.xml b/fizz-common/pom.xml index 71294c2..1d584a9 100644 --- a/fizz-common/pom.xml +++ b/fizz-common/pom.xml @@ -5,7 +5,7 @@ fizz-gateway-community com.fizzgate - 2.2.0-beta6 + 2.2.0-beta7 ../pom.xml 4.0.0 diff --git a/fizz-core/pom.xml b/fizz-core/pom.xml index 317819f..b6ed397 100644 --- a/fizz-core/pom.xml +++ b/fizz-core/pom.xml @@ -5,7 +5,7 @@ fizz-gateway-community com.fizzgate - 2.2.0-beta6 + 2.2.0-beta7 ../pom.xml 4.0.0 diff --git a/fizz-core/src/main/java/we/filter/FlowControlFilter.java b/fizz-core/src/main/java/we/filter/FlowControlFilter.java index af2001c..655d32a 100644 --- a/fizz-core/src/main/java/we/filter/FlowControlFilter.java +++ b/fizz-core/src/main/java/we/filter/FlowControlFilter.java @@ -285,22 +285,28 @@ public class FlowControlFilter extends FizzWebFilter { prev = resourceConfigs.get(sz - 1).getResourceId(); prevPrev = resourceConfigs.get(sz - 2).getResourceId(); - if (rateLimitConfig.type == ResourceRateLimitConfig.Type.APP && rateLimitConfig.path != null) { - String app = ResourceRateLimitConfig.getApp(prev); - if (app == null) { - something4(resourceConfigs, rateLimitConfig.app, null, null); - something4(resourceConfigs, rateLimitConfig.app, null, rateLimitConfig.service); - } else { - String service = ResourceRateLimitConfig.getService(prev); - if (service == null) { - something4(resourceConfigs, rateLimitConfig.app, null, rateLimitConfig.service); + if (rateLimitConfig.type == ResourceRateLimitConfig.Type.APP) { + String app = ResourceRateLimitConfig.getApp(prev); + if (rateLimitConfig.path == null) { + if (rateLimitConfig.service != null && app == null) { + something4(resourceConfigs, rateLimitConfig.app, null, null); + } } else { - app = ResourceRateLimitConfig.getApp(prevPrev); - if (app == null) { - something4(resourceConfigs, rateLimitConfig.app, null, null); - } + if (app == null) { + something4(resourceConfigs, rateLimitConfig.app, null, null); + something4(resourceConfigs, rateLimitConfig.app, null, rateLimitConfig.service); + } else { + String service = ResourceRateLimitConfig.getService(prev); + if (service == null) { + something4(resourceConfigs, rateLimitConfig.app, null, rateLimitConfig.service); + } else { + app = ResourceRateLimitConfig.getApp(prevPrev); + if (app == null) { + something4(resourceConfigs, rateLimitConfig.app, null, null); + } + } + } } - } } else if (rateLimitConfig.type == ResourceRateLimitConfig.Type.IP) { diff --git a/fizz-plugin/pom.xml b/fizz-plugin/pom.xml index 6079cba..c6bff1e 100644 --- a/fizz-plugin/pom.xml +++ b/fizz-plugin/pom.xml @@ -5,7 +5,7 @@ fizz-gateway-community com.fizzgate - 2.2.0-beta6 + 2.2.0-beta7 ../pom.xml 4.0.0 diff --git a/fizz-spring-boot-starter/pom.xml b/fizz-spring-boot-starter/pom.xml index 37a19b0..1d49a14 100644 --- a/fizz-spring-boot-starter/pom.xml +++ b/fizz-spring-boot-starter/pom.xml @@ -5,7 +5,7 @@ fizz-gateway-community com.fizzgate - 2.2.0-beta6 + 2.2.0-beta7 ../pom.xml 4.0.0 diff --git a/pom.xml b/pom.xml index 7f8c28b..34a0752 100644 --- a/pom.xml +++ b/pom.xml @@ -31,7 +31,7 @@ fizz-gateway-community ${project.artifactId} fizz gateway community - 2.2.0-beta6 + 2.2.0-beta7 pom fizz-common From b3c095994638695cbd1f61cc217f24ab8f5f8892 Mon Sep 17 00:00:00 2001 From: hongqiaowei Date: Tue, 6 Jul 2021 18:53:57 +0800 Subject: [PATCH 19/25] Release 2.2.0-beta8 --- fizz-bootstrap/pom.xml | 2 +- fizz-common/pom.xml | 2 +- fizz-core/pom.xml | 2 +- fizz-core/src/main/java/we/filter/FlowControlFilter.java | 8 ++++---- fizz-plugin/pom.xml | 2 +- fizz-spring-boot-starter/pom.xml | 2 +- pom.xml | 2 +- 7 files changed, 10 insertions(+), 10 deletions(-) diff --git a/fizz-bootstrap/pom.xml b/fizz-bootstrap/pom.xml index 8b6a76e..a4c7142 100644 --- a/fizz-bootstrap/pom.xml +++ b/fizz-bootstrap/pom.xml @@ -12,7 +12,7 @@ com.fizzgate fizz-bootstrap - 2.2.0-beta7 + 2.2.0-beta8 1.8 diff --git a/fizz-common/pom.xml b/fizz-common/pom.xml index 1d584a9..e3276c1 100644 --- a/fizz-common/pom.xml +++ b/fizz-common/pom.xml @@ -5,7 +5,7 @@ fizz-gateway-community com.fizzgate - 2.2.0-beta7 + 2.2.0-beta8 ../pom.xml 4.0.0 diff --git a/fizz-core/pom.xml b/fizz-core/pom.xml index b6ed397..b190420 100644 --- a/fizz-core/pom.xml +++ b/fizz-core/pom.xml @@ -5,7 +5,7 @@ fizz-gateway-community com.fizzgate - 2.2.0-beta7 + 2.2.0-beta8 ../pom.xml 4.0.0 diff --git a/fizz-core/src/main/java/we/filter/FlowControlFilter.java b/fizz-core/src/main/java/we/filter/FlowControlFilter.java index 655d32a..7df67c3 100644 --- a/fizz-core/src/main/java/we/filter/FlowControlFilter.java +++ b/fizz-core/src/main/java/we/filter/FlowControlFilter.java @@ -111,10 +111,10 @@ public class FlowControlFilter extends FizzWebFilter { return WebUtils.buildJsonDirectResponse(exchange, HttpStatus.FORBIDDEN, null, json); } String app = WebUtils.getAppId(exchange); - if (app != null && !appService.getAppMap().containsKey(app)) { - String json = RespEntity.toJson(HttpStatus.FORBIDDEN.value(), "no app " + app, exchange.getRequest().getId()); - return WebUtils.buildJsonDirectResponse(exchange, HttpStatus.FORBIDDEN, null, json); - } + // if (app != null && !appService.getAppMap().containsKey(app)) { + // String json = RespEntity.toJson(HttpStatus.FORBIDDEN.value(), "no app " + app, exchange.getRequest().getId()); + // return WebUtils.buildJsonDirectResponse(exchange, HttpStatus.FORBIDDEN, null, json); + // } path = WebUtils.getClientReqPath(exchange); String ip = WebUtils.getOriginIp(exchange); diff --git a/fizz-plugin/pom.xml b/fizz-plugin/pom.xml index c6bff1e..5a95aec 100644 --- a/fizz-plugin/pom.xml +++ b/fizz-plugin/pom.xml @@ -5,7 +5,7 @@ fizz-gateway-community com.fizzgate - 2.2.0-beta7 + 2.2.0-beta8 ../pom.xml 4.0.0 diff --git a/fizz-spring-boot-starter/pom.xml b/fizz-spring-boot-starter/pom.xml index 1d49a14..4ef3aab 100644 --- a/fizz-spring-boot-starter/pom.xml +++ b/fizz-spring-boot-starter/pom.xml @@ -5,7 +5,7 @@ fizz-gateway-community com.fizzgate - 2.2.0-beta7 + 2.2.0-beta8 ../pom.xml 4.0.0 diff --git a/pom.xml b/pom.xml index 34a0752..213ce5b 100644 --- a/pom.xml +++ b/pom.xml @@ -31,7 +31,7 @@ fizz-gateway-community ${project.artifactId} fizz gateway community - 2.2.0-beta7 + 2.2.0-beta8 pom fizz-common From fa669a8e2be4e49a8165b99a59b9b79cbe0d99ba Mon Sep 17 00:00:00 2001 From: hongqiaowei Date: Wed, 7 Jul 2021 14:20:01 +0800 Subject: [PATCH 20/25] Release 2.2.0-beta9 --- fizz-bootstrap/pom.xml | 2 +- fizz-common/pom.xml | 2 +- fizz-core/pom.xml | 2 +- .../main/java/we/filter/PreprocessFilter.java | 32 +++++++++++++++---- .../java/we/plugin/FixedPluginFilter.java | 1 + fizz-plugin/pom.xml | 2 +- fizz-spring-boot-starter/pom.xml | 2 +- pom.xml | 2 +- 8 files changed, 33 insertions(+), 12 deletions(-) diff --git a/fizz-bootstrap/pom.xml b/fizz-bootstrap/pom.xml index a4c7142..8db531c 100644 --- a/fizz-bootstrap/pom.xml +++ b/fizz-bootstrap/pom.xml @@ -12,7 +12,7 @@ com.fizzgate fizz-bootstrap - 2.2.0-beta8 + 2.2.0-beta9 1.8 diff --git a/fizz-common/pom.xml b/fizz-common/pom.xml index e3276c1..6593347 100644 --- a/fizz-common/pom.xml +++ b/fizz-common/pom.xml @@ -5,7 +5,7 @@ fizz-gateway-community com.fizzgate - 2.2.0-beta8 + 2.2.0-beta9 ../pom.xml 4.0.0 diff --git a/fizz-core/pom.xml b/fizz-core/pom.xml index b190420..c70b2de 100644 --- a/fizz-core/pom.xml +++ b/fizz-core/pom.xml @@ -5,7 +5,7 @@ fizz-gateway-community com.fizzgate - 2.2.0-beta8 + 2.2.0-beta9 ../pom.xml 4.0.0 diff --git a/fizz-core/src/main/java/we/filter/PreprocessFilter.java b/fizz-core/src/main/java/we/filter/PreprocessFilter.java index ac7329f..82e9dfd 100644 --- a/fizz-core/src/main/java/we/filter/PreprocessFilter.java +++ b/fizz-core/src/main/java/we/filter/PreprocessFilter.java @@ -17,15 +17,13 @@ package we.filter; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.context.ApplicationContext; import org.springframework.core.annotation.Order; import org.springframework.http.HttpStatus; import org.springframework.stereotype.Component; import org.springframework.web.server.ServerWebExchange; import org.springframework.web.server.WebFilterChain; import reactor.core.publisher.Mono; +import we.plugin.FixedPluginFilter; import we.plugin.FizzPluginFilterChain; import we.plugin.PluginFilter; import we.plugin.auth.ApiConfig; @@ -37,6 +35,7 @@ import we.util.WebUtils; import javax.annotation.Resource; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.function.Function; @@ -75,15 +74,22 @@ public class PreprocessFilter extends FizzWebFilter { if (authRes instanceof ApiConfig) { ApiConfig ac = (ApiConfig) authRes; afterAuth(exchange, ac); + m = executeFixedPluginFilters(exchange); + m = m.defaultIfEmpty(ReactorUtils.NULL); if (ac.pluginConfigs == null || ac.pluginConfigs.isEmpty()) { return m.flatMap(func(exchange, chain)); } else { - eas.put(FizzPluginFilterChain.WEB_FILTER_CHAIN, chain); - return FizzPluginFilterChain.next(exchange); + return m.flatMap( + nil -> { + eas.put(FizzPluginFilterChain.WEB_FILTER_CHAIN, chain); + return FizzPluginFilterChain.next(exchange); + } + ); } } else if (authRes == ApiConfigService.Access.YES) { afterAuth(exchange, null); - return m.flatMap(func(exchange, chain)); + m = executeFixedPluginFilters(exchange); + return m.defaultIfEmpty(ReactorUtils.NULL).flatMap(func(exchange, chain)); } else { String err = null; if (authRes instanceof ApiConfigService.Access) { @@ -138,4 +144,18 @@ public class PreprocessFilter extends FizzWebFilter { return chain.filter(exchange); }; } + + private Mono executeFixedPluginFilters(ServerWebExchange exchange) { + Mono vm = Mono.empty(); + List fixedPluginFilters = FixedPluginFilter.getPluginFilters(); + for (byte i = 0; i < fixedPluginFilters.size(); i++) { + FixedPluginFilter fpf = fixedPluginFilters.get(i); + vm = vm.defaultIfEmpty(ReactorUtils.NULL).flatMap( + v -> { + return fpf.filter(exchange, null, null); + } + ); + } + return vm; + } } diff --git a/fizz-core/src/main/java/we/plugin/FixedPluginFilter.java b/fizz-core/src/main/java/we/plugin/FixedPluginFilter.java index 02ea426..fd51e70 100644 --- a/fizz-core/src/main/java/we/plugin/FixedPluginFilter.java +++ b/fizz-core/src/main/java/we/plugin/FixedPluginFilter.java @@ -29,6 +29,7 @@ import java.util.*; * @author hongqiaowei */ +@Deprecated public abstract class FixedPluginFilter extends PluginFilter { private static final Logger log = LoggerFactory.getLogger(FixedPluginFilter.class); diff --git a/fizz-plugin/pom.xml b/fizz-plugin/pom.xml index 5a95aec..995f2c8 100644 --- a/fizz-plugin/pom.xml +++ b/fizz-plugin/pom.xml @@ -5,7 +5,7 @@ fizz-gateway-community com.fizzgate - 2.2.0-beta8 + 2.2.0-beta9 ../pom.xml 4.0.0 diff --git a/fizz-spring-boot-starter/pom.xml b/fizz-spring-boot-starter/pom.xml index 4ef3aab..41986b2 100644 --- a/fizz-spring-boot-starter/pom.xml +++ b/fizz-spring-boot-starter/pom.xml @@ -5,7 +5,7 @@ fizz-gateway-community com.fizzgate - 2.2.0-beta8 + 2.2.0-beta9 ../pom.xml 4.0.0 diff --git a/pom.xml b/pom.xml index 213ce5b..be1fb34 100644 --- a/pom.xml +++ b/pom.xml @@ -31,7 +31,7 @@ fizz-gateway-community ${project.artifactId} fizz gateway community - 2.2.0-beta8 + 2.2.0-beta9 pom fizz-common From 43c1739b3334efd29345ceedd0365d206f84f2a4 Mon Sep 17 00:00:00 2001 From: hongqiaowei Date: Wed, 7 Jul 2021 17:01:15 +0800 Subject: [PATCH 21/25] PluginConfig.FIXED_CONFIG => PluginConfig.CUSTOM_CONFIG --- fizz-core/src/main/java/we/plugin/PluginConfig.java | 4 ++-- fizz-core/src/main/java/we/plugin/PluginFilter.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/fizz-core/src/main/java/we/plugin/PluginConfig.java b/fizz-core/src/main/java/we/plugin/PluginConfig.java index 6ad8779..7a7dfea 100644 --- a/fizz-core/src/main/java/we/plugin/PluginConfig.java +++ b/fizz-core/src/main/java/we/plugin/PluginConfig.java @@ -30,7 +30,7 @@ import java.util.Map; public class PluginConfig { - public static final String FIXED_CONFIG = "$fc"; + public static final String CUSTOM_CONFIG = "$fc"; public String plugin; // tb_plugin.eng_name @@ -55,7 +55,7 @@ public class PluginConfig { if (config == Collections.EMPTY_MAP) { config = new HashMap<>(); } - config.put(FIXED_CONFIG, fixedConfig); + config.put(CUSTOM_CONFIG, fixedConfig); } } diff --git a/fizz-core/src/main/java/we/plugin/PluginFilter.java b/fizz-core/src/main/java/we/plugin/PluginFilter.java index 298c95e..bbef439 100644 --- a/fizz-core/src/main/java/we/plugin/PluginFilter.java +++ b/fizz-core/src/main/java/we/plugin/PluginFilter.java @@ -45,7 +45,7 @@ public abstract class PluginFilter implements FizzPluginFilter { @Override public Mono filter(ServerWebExchange exchange, Map config) { - String fixedConfig = (String) config.get(PluginConfig.FIXED_CONFIG); + String fixedConfig = (String) config.get(PluginConfig.CUSTOM_CONFIG); return filter(exchange, config, fixedConfig); } From f0f0599d9c27204371822f91ad73a6ac948ea900 Mon Sep 17 00:00:00 2001 From: Francis Dong Date: Thu, 8 Jul 2021 12:13:25 +0800 Subject: [PATCH 22/25] Add peak QPS of time window #240 --- fizz-core/src/main/java/we/stats/ResourceStat.java | 3 +++ .../src/main/java/we/stats/TimeWindowStat.java | 13 +++++++++++++ 2 files changed, 16 insertions(+) diff --git a/fizz-core/src/main/java/we/stats/ResourceStat.java b/fizz-core/src/main/java/we/stats/ResourceStat.java index 92b3652..752ba60 100644 --- a/fizz-core/src/main/java/we/stats/ResourceStat.java +++ b/fizz-core/src/main/java/we/stats/ResourceStat.java @@ -203,6 +203,7 @@ public class ResourceStat { long totalReqs = 0; long totalRt = 0; long peakConcurrences = 0; + long peakRps = 0; long errors = 0; long blockReqs = 0; long totalBlockReqs = 0; @@ -215,6 +216,7 @@ public class ResourceStat { peakConcurrences = timeSlot.getPeakConcurrentRequests() > peakConcurrences ? timeSlot.getPeakConcurrentRequests() : peakConcurrences; + peakRps = timeSlot.getCounter().get() > peakRps ? timeSlot.getCounter().get() : peakRps; totalReqs = totalReqs + timeSlot.getCounter().get(); totalRt = totalRt + timeSlot.getTotalRt().get(); errors = errors + timeSlot.getErrors().get(); @@ -232,6 +234,7 @@ public class ResourceStat { tws.setBlockRequests(blockReqs); tws.setTotalBlockRequests(totalBlockReqs); tws.setCompReqs(compReqs); + tws.setPeakRps(new BigDecimal(peakRps)); if (compReqs > 0) { tws.setAvgRt(totalRt / compReqs); diff --git a/fizz-core/src/main/java/we/stats/TimeWindowStat.java b/fizz-core/src/main/java/we/stats/TimeWindowStat.java index 2204208..50f35b7 100644 --- a/fizz-core/src/main/java/we/stats/TimeWindowStat.java +++ b/fizz-core/src/main/java/we/stats/TimeWindowStat.java @@ -69,6 +69,11 @@ public class TimeWindowStat { * the average RPS(Requests Per Second) of time window */ private BigDecimal rps; + + /** + * the peak RPS(Requests Per Second) of time window + */ + private BigDecimal peakRps; /** * Peak concurrent requests of the time window @@ -181,4 +186,12 @@ public class TimeWindowStat { this.totalBlockRequests = totalBlockRequests; } + public BigDecimal getPeakRps() { + return peakRps; + } + + public void setPeakRps(BigDecimal peakRps) { + this.peakRps = peakRps; + } + } From 520e06f6a3bbe67b635803bf984988171f4d9ad2 Mon Sep 17 00:00:00 2001 From: hongqiaowei Date: Thu, 8 Jul 2021 14:26:54 +0800 Subject: [PATCH 23/25] add peak rps to flow stat --- .../src/main/java/we/config/FlowStatSchedConfig.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/fizz-core/src/main/java/we/config/FlowStatSchedConfig.java b/fizz-core/src/main/java/we/config/FlowStatSchedConfig.java index a4034af..7f08db3 100644 --- a/fizz-core/src/main/java/we/config/FlowStatSchedConfig.java +++ b/fizz-core/src/main/java/we/config/FlowStatSchedConfig.java @@ -72,6 +72,7 @@ public class FlowStatSchedConfig extends SchedConfig { private static final String _sourceIp = "\"sourceIp\":"; private static final String _service = "\"service\":"; private static final String _path = "\"path\":"; + private static final String _peakRps = "\"peakRps\":"; private static final String parentResourceList = "$prl"; @@ -183,12 +184,18 @@ public class FlowStatSchedConfig extends SchedConfig { StringBuilder b = ThreadContext.getStringBuilder(); long timeWin = w.getStartTime(); BigDecimal rps = w.getRps(); - double qps; + BigDecimal peakRps = w.getPeakRps(); + double qps, pRps; if (rps == null) { qps = 0.00; } else { qps = rps.doubleValue(); } + if (peakRps == null) { + pRps = 0.00; + } else { + pRps = peakRps.doubleValue(); + } // AtomicLong totalBlockRequests = resourceTimeWindow2totalBlockRequestsMap.get(resource + timeWin); // long tbrs = (totalBlockRequests == null ? w.getBlockRequests() : w.getBlockRequests() + totalBlockRequests.longValue()); @@ -231,6 +238,7 @@ public class FlowStatSchedConfig extends SchedConfig { b.append(_completeReqs); b.append(w.getCompReqs()); b.append(Constants.Symbol.COMMA); b.append(_peakConcurrents); b.append(w.getPeakConcurrentReqeusts()); b.append(Constants.Symbol.COMMA); b.append(_reqPerSec); b.append(qps); b.append(Constants.Symbol.COMMA); + b.append(_peakRps); b.append(pRps); b.append(Constants.Symbol.COMMA); b.append(_blockReqs); b.append(w.getBlockRequests()); b.append(Constants.Symbol.COMMA); b.append(_totalBlockReqs); b.append(tbrs); b.append(Constants.Symbol.COMMA); b.append(_errors); b.append(w.getErrors()); b.append(Constants.Symbol.COMMA); From c32c2068728afa2a6ba8a213e84c67b9b2ff42b1 Mon Sep 17 00:00:00 2001 From: Francis Dong Date: Mon, 5 Jul 2021 15:30:18 +0800 Subject: [PATCH 24/25] Support process control in aggregation #215 --- fizz-core/src/main/java/we/fizz/Pipeline.java | 26 +- fizz-core/src/main/java/we/fizz/Step.java | 48 ++- .../src/main/java/we/fizz/StepContext.java | 159 +++++++++ .../src/main/java/we/fizz/StepResponse.java | 33 ++ .../we/fizz/component/ComponentHelper.java | 159 +++++++++ .../we/fizz/component/ComponentTypeEnum.java | 50 +++ .../we/fizz/component/FixedDataTypeEnum.java | 44 +++ .../java/we/fizz/component/IComponent.java | 34 ++ .../java/we/fizz/component/OperatorEnum.java | 38 ++ .../we/fizz/component/RefDataTypeEnum.java | 43 +++ .../fizz/component/StepContextPosition.java | 47 +++ .../java/we/fizz/component/ValueTypeEnum.java | 41 +++ .../java/we/fizz/component/circle/Circle.java | 280 +++++++++++++++ .../we/fizz/component/circle/CircleItem.java | 37 ++ .../fizz/component/condition/Condition.java | 262 ++++++++++++++ .../component/condition/ConditionValue.java | 58 ++++ .../fizz/exception/FizzRuntimeException.java | 10 +- .../main/java/we/fizz/input/InputConfig.java | 14 +- .../main/java/we/fizz/input/InputFactory.java | 3 + .../main/java/we/fizz/input/PathMapping.java | 1 + .../input/extension/dubbo/DubboInput.java | 2 +- .../fizz/input/extension/grpc/GrpcInput.java | 2 +- .../input/extension/request/RequestInput.java | 2 +- .../java/we/fizz/component/CircleTests.java | 170 +++++++++ .../we/fizz/component/ConditionTests.java | 327 ++++++++++++++++++ .../java/we/fizz/input/PathMappingTests.java | 27 ++ 26 files changed, 1900 insertions(+), 17 deletions(-) create mode 100644 fizz-core/src/main/java/we/fizz/component/ComponentHelper.java create mode 100644 fizz-core/src/main/java/we/fizz/component/ComponentTypeEnum.java create mode 100644 fizz-core/src/main/java/we/fizz/component/FixedDataTypeEnum.java create mode 100644 fizz-core/src/main/java/we/fizz/component/IComponent.java create mode 100644 fizz-core/src/main/java/we/fizz/component/OperatorEnum.java create mode 100644 fizz-core/src/main/java/we/fizz/component/RefDataTypeEnum.java create mode 100644 fizz-core/src/main/java/we/fizz/component/StepContextPosition.java create mode 100644 fizz-core/src/main/java/we/fizz/component/ValueTypeEnum.java create mode 100644 fizz-core/src/main/java/we/fizz/component/circle/Circle.java create mode 100644 fizz-core/src/main/java/we/fizz/component/circle/CircleItem.java create mode 100644 fizz-core/src/main/java/we/fizz/component/condition/Condition.java create mode 100644 fizz-core/src/main/java/we/fizz/component/condition/ConditionValue.java create mode 100644 fizz-core/src/test/java/we/fizz/component/CircleTests.java create mode 100644 fizz-core/src/test/java/we/fizz/component/ConditionTests.java diff --git a/fizz-core/src/main/java/we/fizz/Pipeline.java b/fizz-core/src/main/java/we/fizz/Pipeline.java index 274a2e5..db99759 100644 --- a/fizz-core/src/main/java/we/fizz/Pipeline.java +++ b/fizz-core/src/main/java/we/fizz/Pipeline.java @@ -43,6 +43,9 @@ import reactor.core.publisher.Flux; import reactor.core.publisher.Mono; import we.constants.CommonConstants; import we.exception.ExecuteScriptException; +import we.fizz.component.ComponentHelper; +import we.fizz.component.IComponent; +import we.fizz.component.StepContextPosition; import we.fizz.input.ClientInputConfig; import we.fizz.input.Input; import we.fizz.input.InputConfig; @@ -116,14 +119,12 @@ public class Pipeline { }else { LinkedList opSteps = (LinkedList) steps.clone(); Step step1 = opSteps.removeFirst(); - step1.beforeRun(stepContext, null); - Mono> result = createStep(step1).expand(response -> { + Mono> result = runStep(step1, null).expand(response -> { if (opSteps.isEmpty() || response.isStop()) { return Mono.empty(); } Step step = opSteps.pop(); - step.beforeRun(stepContext, response); - return createStep(step); + return runStep(step, response); }).flatMap(response -> Flux.just(response)).collectList(); return result.flatMap(clientResponse -> { return handleOutput(input); @@ -131,6 +132,23 @@ public class Pipeline { } } + private Mono runStep(Step step, StepResponse response){ + List components = step.getComponents(); + if (components != null && components.size() > 0) { + StepContextPosition stepCtxPos = new StepContextPosition(step.getName()); + return ComponentHelper.run(components, stepContext, stepCtxPos, (ctx, pos) -> { + step.beforeRun(stepContext, null); + return createStep(step).flatMap(r -> { + ctx.addStepCircleResult(pos.getStepName()); + return Mono.just(r); + }); + }).flatMap(sr -> Mono.just((StepResponse)sr)); + } else { + step.beforeRun(stepContext, null); + return createStep(step); + } + } + private Mono handleOutput(Input input){ // 数据转换 long t3 = System.currentTimeMillis(); diff --git a/fizz-core/src/main/java/we/fizz/Step.java b/fizz-core/src/main/java/we/fizz/Step.java index d589ba1..2a7e9e0 100644 --- a/fizz-core/src/main/java/we/fizz/Step.java +++ b/fizz-core/src/main/java/we/fizz/Step.java @@ -22,7 +22,11 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collector; +import java.util.stream.Collectors; +import org.noear.snack.ONode; import org.springframework.context.ConfigurableApplicationContext; import org.springframework.http.HttpMethod; import org.springframework.http.MediaType; @@ -34,6 +38,12 @@ import org.springframework.web.reactive.function.client.WebClient; import com.alibaba.fastjson.JSON; import reactor.core.publisher.Mono; +import we.fizz.component.ComponentHelper; +import we.fizz.component.ComponentTypeEnum; +import we.fizz.component.IComponent; +import we.fizz.component.StepContextPosition; +import we.fizz.component.circle.Circle; +import we.fizz.component.condition.Condition; import we.fizz.input.Input; import we.fizz.input.InputConfig; import we.fizz.input.InputContext; @@ -56,6 +66,16 @@ public class Step { private Map dataMapping; private Map requestConfigs = new HashMap(); + + private List components; + + public List getComponents() { + return components; + } + + public void setComponents(List components) { + this.components = components; + } public SoftReference getWeakPipeline() { return weakPipeline; @@ -78,6 +98,7 @@ public class Step { InputConfig inputConfig = InputFactory.createInputConfig(requestConfig); step.addRequestConfig((String)requestConfig.get("name"), inputConfig); } + step.setComponents(ComponentHelper.buildComponents((List>) config.get("components"))); return step; } } @@ -112,15 +133,32 @@ public class Step { public List run() { List monos = new ArrayList(); - for(String name :inputs.keySet()) { - Input input = inputs.get(name); - if (input.needRun(stepContext)) { - Mono singleMono = input.run(); - monos.add(singleMono); + for(String requestName :inputs.keySet()) { + Input input = inputs.get(requestName); + List components = input.getConfig().getComponents(); + if (components != null && components.size() > 0) { + StepContextPosition stepCtxPos = new StepContextPosition(name, requestName); + Mono result = ComponentHelper.run(components, stepContext, stepCtxPos, (ctx, pos) -> { + if (input.needRun(ctx)) { + return input.run().flatMap(r -> { + ctx.addRequestCircleResult(pos.getStepName(), pos.getRequestName()); + return Mono.just(r); + }); + } + return Mono.just(new HashMap()); + }); + monos.add(result); + } else { + if (input.needRun(stepContext)) { + Mono singleMono = input.run(); + monos.add(singleMono); + } } } return monos; } + + public void afeterRun() { diff --git a/fizz-core/src/main/java/we/fizz/StepContext.java b/fizz-core/src/main/java/we/fizz/StepContext.java index 4684985..6e6bf84 100644 --- a/fizz-core/src/main/java/we/fizz/StepContext.java +++ b/fizz-core/src/main/java/we/fizz/StepContext.java @@ -26,6 +26,8 @@ import java.util.concurrent.ConcurrentHashMap; import org.springframework.context.ConfigurableApplicationContext; import org.springframework.http.codec.multipart.FilePart; +import com.alibaba.fastjson.JSON; + import we.constants.CommonConstants; /** @@ -146,6 +148,8 @@ public class StepContext extends ConcurrentHashMap { requests = new HashMap<>(); stepResponse.setRequests(requests); requests.put(requestName, new HashMap()); + }else if(!requests.containsKey(requestName)) { + requests.put(requestName, new HashMap()); } return (Map) requests.get(requestName); } @@ -681,6 +685,161 @@ public class StepContext extends ConcurrentHashMap { } return request.get(key); } + + /** + * 设置Step的循环对象
+ * Set the current circle item of step
+ *
+ * + * @param stepName + * @param item + * @param index + */ + public void setStepCircleItem(String stepName, Object item, Integer index) { + StepResponse stepResponse = (StepResponse) this.get(stepName); + if (stepResponse == null) { + return; + } + stepResponse.setItem(item); + stepResponse.setIndex(index); + } + + /** + * 添加Step的循环结果
+ * Add the result of current circle item
+ *
+ * + * @param stepName + * @param key + * @param value + */ + public void addStepCircleResult(String stepName) { + StepResponse stepResponse = (StepResponse) this.get(stepName); + if (stepResponse == null) { + return; + } + List> circle = (List>) stepResponse.getCircle(); + if (circle == null) { + circle = new ArrayList<>(); + stepResponse.setCircle(circle); + } + Map circleResult = new HashMap<>(); + circleResult.put("requests", deepCopy(stepResponse.getRequests())); + circleResult.put("result", deepCopy(stepResponse.getResult())); + circleResult.put("item", deepCopy(stepResponse.getItem())); + circleResult.put("index", stepResponse.getIndex()); + circle.add(circleResult); + } + + /** + * 获取Step的循环对象
+ * Returns current circle item
+ * + * @param stepName + */ + public Object getStepItem(String stepName) { + StepResponse stepResponse = (StepResponse) this.get(stepName); + if (stepResponse == null) { + return null; + } + return stepResponse.getItem(); + } + + /** + * 获取Step的循环结果
+ * Returns circle result list of step
+ * + * @param stepName + */ + public List> getStepCircle(String stepName) { + StepResponse stepResponse = (StepResponse) this.get(stepName); + if (stepResponse == null) { + return null; + } + return stepResponse.getCircle(); + } + + /** + * 设置请求的循环对象
+ * Set current circle item of request
+ * + * @param stepName + * @param requestName + * @param item + */ + public void setRequestCircleItem(String stepName, String requestName, Object item, Integer index) { + Map request = getStepRequest(stepName, requestName); + if (request == null) { + return; + } + request.put("item", item); + request.put("index", index); + } + + /** + * 设置请求的循环结果
+ * Set current circle result of request
+ * + * @param stepName + * @param requestName + */ + public void addRequestCircleResult(String stepName, String requestName) { + Map request = getStepRequest(stepName, requestName); + if (request == null) { + return; + } + List> circle = (List>) request.get("circle"); + if (circle == null) { + circle = new ArrayList<>(); + request.put("circle", circle); + } + Map circleResult = new HashMap<>(); + circleResult.put("request", deepCopy(request.get("request"))); + circleResult.put("response", deepCopy(request.get("response"))); + circleResult.put("item", deepCopy(request.get("item"))); + circleResult.put("index", request.get("index")); + circle.add(circleResult); + } + + /** + * 获取请求的循环对象
+ * Returns the current circle item of request
+ * + * @param stepName + * @param requestName + */ + public List> getRequestCircleItem(String stepName, String requestName) { + Map request = getStepRequest(stepName, requestName); + if (request == null) { + return null; + } + return (List>) request.get("circle"); + } + + /** + * 获取请求的循环结果
+ * Returns circle result list of request
+ * + * @param stepName + * @param requestName + */ + public Object getRequestCircle(String stepName, String requestName) { + Map request = getStepRequest(stepName, requestName); + if (request == null) { + return null; + } + return request.get("item"); + } + + private Object deepCopy(Object obj) { + if(obj == null) { + return obj; + } + if(obj.getClass().isPrimitive()) { + return obj; + } + return JSON.parse(JSON.toJSONString(obj)); + } public ConfigurableApplicationContext getApplicationContext(){ return this.applicationContext; diff --git a/fizz-core/src/main/java/we/fizz/StepResponse.java b/fizz-core/src/main/java/we/fizz/StepResponse.java index 70e2230..01dc264 100644 --- a/fizz-core/src/main/java/we/fizz/StepResponse.java +++ b/fizz-core/src/main/java/we/fizz/StepResponse.java @@ -18,6 +18,7 @@ package we.fizz; import java.util.HashMap; +import java.util.List; import java.util.Map; /** @@ -28,6 +29,12 @@ public class StepResponse { private Map> requests; private Map result; private boolean stop; + // circle item + private Object item; + // index of circle item + private Integer index; + // circle results + private List> circle; public StepResponse(Step aStep, HashMap item, Map> requests) { setStepName(aStep.getName()); @@ -39,6 +46,14 @@ public class StepResponse { setResult(item); } + public void addRequest(String requestName, Map requestObj) { + if (this.requests.containsKey(requestName)) { + this.requests.get(requestName).putAll(requestObj); + } else { + this.requests.put(requestName, requestObj); + } + } + public boolean isStop() { return stop; } @@ -63,5 +78,23 @@ public class StepResponse { public void setResult(Map result) { this.result = result; } + public Object getItem() { + return item; + } + public void setItem(Object item) { + this.item = item; + } + public List> getCircle() { + return circle; + } + public void setCircle(List> circle) { + this.circle = circle; + } + public Integer getIndex() { + return index; + } + public void setIndex(Integer index) { + this.index = index; + } } diff --git a/fizz-core/src/main/java/we/fizz/component/ComponentHelper.java b/fizz-core/src/main/java/we/fizz/component/ComponentHelper.java new file mode 100644 index 0000000..72f1eb6 --- /dev/null +++ b/fizz-core/src/main/java/we/fizz/component/ComponentHelper.java @@ -0,0 +1,159 @@ +/* + * Copyright (C) 2021 the original author or authors. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package we.fizz.component; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.function.BiFunction; + +import org.noear.snack.ONode; + +import com.alibaba.fastjson.JSON; + +import reactor.core.publisher.Mono; +import we.fizz.StepContext; +import we.fizz.component.circle.Circle; +import we.fizz.component.condition.Condition; + +/** + * Condition component + * + * @author Francis Dong + * + */ +public class ComponentHelper { + + /** + * Converts step context to ONode + * + * @param stepContext context + * @return + */ + public static ONode toONode(StepContext stepContext) { + ONode o = null; + synchronized (stepContext) { + o = ONode.loadObj(stepContext); + } + return o; + } + + public static List buildComponents(List> componentConfig) { + List components = new ArrayList<>(); + + if (componentConfig != null && componentConfig.size() > 0) { + for (Map m : componentConfig) { + // condition + if (ComponentTypeEnum.CONDITION.getCode().equals(m.get("type"))) { + Condition c = JSON.parseObject(JSON.toJSONString(m), Condition.class); + components.add(c); + } + + // circle + if (ComponentTypeEnum.CIRCLE.getCode().equals(m.get("type"))) { + Circle c = JSON.parseObject(JSON.toJSONString(m), Circle.class); + components.add(c); + } + } + } + + return components; + } + + /** + * + * @param components + * @param stepContext + * @param f + */ + public static Mono run(List components, StepContext stepContext, + StepContextPosition stepCtxPos, BiFunction f) { + if (components != null && components.size() > 0) { + // conditions before circle component + List conditions = new ArrayList<>(); + Circle circle = null; + for (IComponent component : components) { + if (ComponentTypeEnum.CIRCLE == component.getType()) { + circle = (Circle) component; + } + if (circle == null && ComponentTypeEnum.CONDITION == component.getType()) { + conditions.add((Condition) component); + } + } + + if (conditions != null && conditions.size() > 0) { + ONode ctxNode = toONode(stepContext); + for (Condition c : conditions) { + if (!c.exec(ctxNode)) { + return null; + } + } + } + + if (circle != null) { + return circle.exec(stepContext, stepCtxPos, f); + } +// // conditions before circle component +// List conditions1 = new ArrayList<>(); +// // conditions after circle component +// List conditions2 = new ArrayList<>(); +// Circle circle = null; +// for (IComponent component : components) { +// if (ComponentTypeEnum.CIRCLE == component.getType()) { +// circle = (Circle) component; +// } +// if (circle == null && ComponentTypeEnum.CONDITION == component.getType()) { +// conditions1.add((Condition) component); +// } +// if (circle != null && ComponentTypeEnum.CONDITION == component.getType()) { +// conditions2.add((Condition) component); +// } +// } +// +// if (conditions1 != null && conditions1.size() > 0) { +// ONode ctxNode = toONode(stepContext); +// for (Condition c : conditions1) { +// if (!c.exec(ctxNode)) { +// return null; +// } +// } +// } +// +// if (circle != null) { +// return circle.exec(stepContext, (ctx) -> { +// boolean canRun = true; +// if (conditions2 != null && conditions2.size() > 0) { +// ONode ctxNode = toONode(ctx); +// for (Condition c : conditions2) { +// if (!c.exec(ctxNode)) { +// canRun = false; +// } +// } +// } +// if (canRun) { +// return f.apply(ctx); +// } else { +// return Mono.empty(); +// } +// +// }); +// } + } + return Mono.empty(); + } + +} diff --git a/fizz-core/src/main/java/we/fizz/component/ComponentTypeEnum.java b/fizz-core/src/main/java/we/fizz/component/ComponentTypeEnum.java new file mode 100644 index 0000000..5f47e20 --- /dev/null +++ b/fizz-core/src/main/java/we/fizz/component/ComponentTypeEnum.java @@ -0,0 +1,50 @@ +/* + * Copyright (C) 2021 the original author or authors. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package we.fizz.component; + +/** + * Component Type + * + * @author Francis Dong + * + */ +public enum ComponentTypeEnum { + + CONDITION("condition"), CIRCLE("circle"); + + private String code; + + private ComponentTypeEnum(String code) { + this.code = code; + } + + public static ComponentTypeEnum getEnumByCode(String code) { + for (ComponentTypeEnum item : ComponentTypeEnum.values()) { + if (item.getCode().equals(code)) { + return item; + } + } + + return null; + } + + public String getCode() { + return code; + } + +} diff --git a/fizz-core/src/main/java/we/fizz/component/FixedDataTypeEnum.java b/fizz-core/src/main/java/we/fizz/component/FixedDataTypeEnum.java new file mode 100644 index 0000000..cf881c7 --- /dev/null +++ b/fizz-core/src/main/java/we/fizz/component/FixedDataTypeEnum.java @@ -0,0 +1,44 @@ +/* + * Copyright (C) 2021 the original author or authors. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package we.fizz.component; + +/** + * Data type of fixed value + * + * @author Francis Dong + * + */ +public enum FixedDataTypeEnum{ + + NUMBER("number"), STRING("string"), BOOLEAN("boolean"); + + private String code; + + private FixedDataTypeEnum(String code) { + this.code = code; + } + + public String getCode() { + return this.code; + } + + public void setCode(String code) { + this.code = code; + } + +} diff --git a/fizz-core/src/main/java/we/fizz/component/IComponent.java b/fizz-core/src/main/java/we/fizz/component/IComponent.java new file mode 100644 index 0000000..65e40cb --- /dev/null +++ b/fizz-core/src/main/java/we/fizz/component/IComponent.java @@ -0,0 +1,34 @@ +/* + * Copyright (C) 2021 the original author or authors. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package we.fizz.component; + +/** + * Component interface + * + * @author Francis Dong + * + */ +public interface IComponent { + + /** + * Returns component type + * @return + */ + public ComponentTypeEnum getType(); + + +} diff --git a/fizz-core/src/main/java/we/fizz/component/OperatorEnum.java b/fizz-core/src/main/java/we/fizz/component/OperatorEnum.java new file mode 100644 index 0000000..2cfce8c --- /dev/null +++ b/fizz-core/src/main/java/we/fizz/component/OperatorEnum.java @@ -0,0 +1,38 @@ +/* + * Copyright (C) 2021 the original author or authors. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package we.fizz.component; + +/** + * Operator + * + * @author Francis Dong + * + */ +public enum OperatorEnum { + + EQ("eq"), NE("ne"), GT("gt"), GE("ge"), LT("lt"), LE("le"), CONTAINS("contains"), NOT_CONTAIN("notContain"), CONTAINS_ANY("containsAny"), + IS_NULL("isNull"), IS_NOT_NULL("isNotNull"), IS_BLANK("isBlank"), IS_NOT_BLANK("isNotBlank"), IS_EMPTY("isEmpty"), + IS_NOT_EMPTY("isNotEmpty"); + + private String code; + + private OperatorEnum(String code) { + this.code = code; + } + +} diff --git a/fizz-core/src/main/java/we/fizz/component/RefDataTypeEnum.java b/fizz-core/src/main/java/we/fizz/component/RefDataTypeEnum.java new file mode 100644 index 0000000..d212a82 --- /dev/null +++ b/fizz-core/src/main/java/we/fizz/component/RefDataTypeEnum.java @@ -0,0 +1,43 @@ +/* + * Copyright (C) 2021 the original author or authors. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package we.fizz.component; + +/** + * Data type of reference value + * + * @author Francis Dong + * + */ +public enum RefDataTypeEnum { + + INT("int"), LONG("long"), FLOAT("float"), DOUBLE("double"), STRING("string"), BOOLEAN("boolean"), ARRAY("array"); + + private String code; + + private RefDataTypeEnum(String code) { + this.code = code; + } + + public String getCode() { + return this.code; + } + + public void setCode(String code) { + this.code = code; + } +} diff --git a/fizz-core/src/main/java/we/fizz/component/StepContextPosition.java b/fizz-core/src/main/java/we/fizz/component/StepContextPosition.java new file mode 100644 index 0000000..9afe709 --- /dev/null +++ b/fizz-core/src/main/java/we/fizz/component/StepContextPosition.java @@ -0,0 +1,47 @@ +/* Copyright (C) 2021 the original author or authors. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package we.fizz.component; + +import lombok.Data; + +/** + * + * @author Francis Dong + * + */ +@Data +public class StepContextPosition { + + private String stepName; + private String requestName; + + public StepContextPosition(String stepName) { + this.stepName = stepName; + } + + public StepContextPosition(String stepName, String requestName) { + this.stepName = stepName; + this.requestName = requestName; + } + + public String getPath() { + if (this.requestName == null) { + return this.stepName; + } + return this.stepName + ".requests." + this.requestName; + } + +} diff --git a/fizz-core/src/main/java/we/fizz/component/ValueTypeEnum.java b/fizz-core/src/main/java/we/fizz/component/ValueTypeEnum.java new file mode 100644 index 0000000..9bb64ac --- /dev/null +++ b/fizz-core/src/main/java/we/fizz/component/ValueTypeEnum.java @@ -0,0 +1,41 @@ +/* + * Copyright (C) 2021 the original author or authors. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package we.fizz.component; + + +/** + * Value Type + * + * @author Francis Dong + * + */ +public enum ValueTypeEnum { + + FIXED("fixed"), REF("ref"); + + private String code; + + private ValueTypeEnum(String code) { + this.code = code; + } + + public String getCode() { + return code; + } + +} diff --git a/fizz-core/src/main/java/we/fizz/component/circle/Circle.java b/fizz-core/src/main/java/we/fizz/component/circle/Circle.java new file mode 100644 index 0000000..85e3e3b --- /dev/null +++ b/fizz-core/src/main/java/we/fizz/component/circle/Circle.java @@ -0,0 +1,280 @@ +/* + * Copyright (C) 2021 the original author or authors. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package we.fizz.component.circle; + +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.function.BiFunction; +import java.util.function.Function; + +import org.noear.snack.ONode; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import lombok.Data; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; +import we.fizz.StepContext; +import we.fizz.component.ComponentHelper; +import we.fizz.component.ComponentTypeEnum; +import we.fizz.component.IComponent; +import we.fizz.component.StepContextPosition; +import we.fizz.component.ValueTypeEnum; +import we.fizz.component.condition.Condition; +import we.fizz.exception.FizzRuntimeException; +import we.fizz.input.PathMapping; +import we.fizz.input.RPCInput; + +/** + * Circle component + * + * @author Francis Dong + * + */ +public class Circle implements IComponent { + + private static final String type = ComponentTypeEnum.CIRCLE.getCode(); + + private String desc; + + private ValueTypeEnum dataSourceType; + + private Object dataSource; + + private List execConditions; + + private List breakConditions; + + @Override + public ComponentTypeEnum getType() { + return ComponentTypeEnum.getEnumByCode(type); + } + + /** + * + * @param desc [optional] description + * @param dataSourceType [required] type of data source + * @param dataSource [required] data source + * @param execConditions [optional] conditions to execute current circle loop + * item + * @param breakConditions [optional] conditions to break circle + */ + public Circle(String desc, ValueTypeEnum dataSourceType, Object dataSource, List execConditions, + List breakConditions) { + this.desc = desc; + this.dataSourceType = dataSourceType; + this.dataSource = dataSource; + this.execConditions = execConditions; + this.breakConditions = breakConditions; + } + + /** + * Current item + */ + private Object currentItem; + + /** + * Index of current item + */ + private Integer index; + + /** + * Fixed value of dataSource + */ + private Integer fixedValue; + + /** + * Reference value of dataSource + */ + private List refValue; + + private boolean refReadFlag; + + private Integer getFixedValue(ONode ctxNode) { + if (fixedValue != null) { + return fixedValue; + } + if (dataSource == null) { + return fixedValue; + } + if (dataSource instanceof Integer || dataSource instanceof Long) { + fixedValue = Integer.valueOf(dataSource.toString()); + if (fixedValue.intValue() < 1) { + throw new FizzRuntimeException("invalid data source, fixed data source must be a positive integer"); + } + return fixedValue; + } else { + throw new FizzRuntimeException("invalid data source, fixed data source must be a positive integer"); + } + } + + @SuppressWarnings("unchecked") + private List getRefValue(ONode ctxNode) { + if (refReadFlag) { + return refValue; + } + Object value = PathMapping.getValueByPath(ctxNode, (String) dataSource); + if (value == null) { + return null; + } + if (value instanceof Collection) { + refValue = (List) value; + return refValue; + } else { + throw new FizzRuntimeException("invalid data source, referenced data source must be a array"); + } + } + + /** + * Returns next circle item, returns null if no item left or dataSource is null + * + * @return + */ + public CircleItem next(ONode ctxNode) { + if (ValueTypeEnum.FIXED.equals(dataSourceType)) { + Integer total = this.getFixedValue(ctxNode); + if (index == null) { + index = 0; + currentItem = index; + return new CircleItem(currentItem, index); + } else if (index.intValue() < total.intValue() - 1) { + index = index + 1; + currentItem = index; + return new CircleItem(currentItem, index); + } else { + return null; + } + } else if (ValueTypeEnum.REF.equals(dataSourceType)) { + List list = this.getRefValue(ctxNode); + if (index == null) { + index = 0; + currentItem = list.get(index); + return new CircleItem(currentItem, index); + } else if (index.intValue() < list.size() - 1) { + index = index + 1; + currentItem = list.get(index); + return new CircleItem(currentItem, index); + } else { + return null; + } + } + return null; + } + + /** + * Returns true if execConditions are all true, false otherwise + * + * @param ctxNode + * @return + */ + public boolean canExec(ONode ctxNode) { + if (this.execConditions != null && this.execConditions.size() > 0) { + try { + for (Condition condition : execConditions) { + if (!condition.exec(ctxNode)) { + return false; + } + } + } catch (FizzRuntimeException e) { + throw new FizzRuntimeException(type + " " + e.getMessage(), e.getCause()); + } + } + return true; + } + + /** + * Returns true if breakConditions are all true, false otherwise + * + * @param ctxNode + * @return + */ + public boolean breakCircle(ONode ctxNode) { + if (this.breakConditions != null && this.breakConditions.size() > 0) { + try { + for (Condition condition : breakConditions) { + if (condition.exec(ctxNode)) { + return true; + } + } + } catch (FizzRuntimeException e) { + throw new FizzRuntimeException(type + " " + e.getMessage(), e.getCause()); + } + } + return false; + } + + @SuppressWarnings("unchecked") + public Mono exec(StepContext stepContext, StepContextPosition stepCtxPos, + BiFunction f) { + ONode ctxNode = ComponentHelper.toONode(stepContext); + CircleItem nextItem = this.next(ctxNode); + if (nextItem != null) { + return Mono.just(new CircleItemResult(nextItem, null)).expand(circleItemResult -> { + // put nextItem to step context and ctxNode for further JSON path mapping + CircleItem cItem = circleItemResult.nextItem; + if (stepCtxPos.getRequestName() != null) { + stepContext.setRequestCircleItem(stepCtxPos.getStepName(), stepCtxPos.getRequestName(), + cItem.getItem(), cItem.getIndex()); + } else { + stepContext.setStepCircleItem(stepCtxPos.getStepName(), cItem.getItem(), cItem.getIndex()); + } + PathMapping.setByPath(ctxNode, stepCtxPos.getPath() + ".item", cItem.getItem(), true); + PathMapping.setByPath(ctxNode, stepCtxPos.getPath() + ".index", cItem.getIndex(), true); + + if (!this.canExec(ctxNode)) { + return Mono.just(new CircleItemResult(this.next(ctxNode), null)); + } + if (this.breakCircle(ctxNode)) { + return Mono.empty(); + } + return f.apply(stepContext, stepCtxPos).flatMap(r -> { + CircleItem nextItem2 = this.next(ctxNode); + if (nextItem2 == null) { + return Mono.empty(); + } + return Mono.just(new CircleItemResult(nextItem2, r)); + }); + }).flatMap(circleItemResult -> Flux.just(circleItemResult)).collectList().flatMap(list -> { + if (list != null && list.size() > 0) { + Collections.reverse(list); + for (int i = 0; i < list.size(); i++) { + if (list.get(i).result != null) { + return Mono.just(list.get(i).result); + } + } + } + return Mono.empty(); + }); + } else { + return Mono.empty(); + } + } + + @Data + class CircleItemResult { + private CircleItem nextItem; + private Object result; + + public CircleItemResult(CircleItem nextItem, Object result) { + this.nextItem = nextItem; + this.result = result; + } + } + +} diff --git a/fizz-core/src/main/java/we/fizz/component/circle/CircleItem.java b/fizz-core/src/main/java/we/fizz/component/circle/CircleItem.java new file mode 100644 index 0000000..34b6cc0 --- /dev/null +++ b/fizz-core/src/main/java/we/fizz/component/circle/CircleItem.java @@ -0,0 +1,37 @@ +/* + * Copyright (C) 2021 the original author or authors. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package we.fizz.component.circle; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * + * @author Francis Dong + * + */ +@Data +@AllArgsConstructor +@NoArgsConstructor +public class CircleItem { + + private Object item; + + private Integer index; + +} diff --git a/fizz-core/src/main/java/we/fizz/component/condition/Condition.java b/fizz-core/src/main/java/we/fizz/component/condition/Condition.java new file mode 100644 index 0000000..a7e62b7 --- /dev/null +++ b/fizz-core/src/main/java/we/fizz/component/condition/Condition.java @@ -0,0 +1,262 @@ +/* + * Copyright (C) 2021 the original author or authors. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package we.fizz.component.condition; + +import java.util.Collection; +import java.util.Map; + +import org.apache.commons.lang3.StringUtils; +import org.noear.snack.ONode; +import org.springframework.util.CollectionUtils; + +import com.alibaba.fastjson.JSON; + +import lombok.AllArgsConstructor; +import lombok.Data; +import we.fizz.component.ComponentTypeEnum; +import we.fizz.component.IComponent; +import we.fizz.component.OperatorEnum; +import we.fizz.component.RefDataTypeEnum; +import we.fizz.component.ValueTypeEnum; +import we.fizz.exception.FizzRuntimeException; +import we.fizz.input.PathMapping; + +/** + * Condition component + * + * @author Francis Dong + * + */ +@Data +@AllArgsConstructor +public class Condition implements IComponent { + + private static final String type = ComponentTypeEnum.CONDITION.getCode(); + + private String desc; + + private ConditionValue value1; + + private OperatorEnum operator; + + private ConditionValue value2; + + @Override + public ComponentTypeEnum getType() { + return ComponentTypeEnum.getEnumByCode(type); + } + + /** + * Execute condition + * + * @return + */ + @SuppressWarnings({ "rawtypes" }) + public boolean exec(ONode ctxNode) { + if (value1 == null || operator == null) { + return false; + } + + boolean rs = false; + try { + Object v1 = null; + if (ValueTypeEnum.FIXED.equals(value1.getType())) { + v1 = value1.getValue(); + } else { + v1 = PathMapping.getValueByPath(ctxNode, (String) value1.getValue()); + v1 = this.cast(value1.getRefDataType(), v1); + } + + Object v2 = null; + if (value2 != null && value2.getType() != null) { + if (ValueTypeEnum.FIXED.equals(value2.getType())) { + v2 = value2.getValue(); + } else { + v2 = PathMapping.getValueByPath(ctxNode, (String) value2.getValue()); + v2 = this.cast(value2.getRefDataType(), v2); + } + } + + switch (operator) { + case EQ: + if (v1 == null && v2 == null) { + rs = true; + } else if (v1 != null && v2 != null) { + rs = this.compare(v1, v2) == 0; + } + break; + case NE: + if (v1 == null && v2 == null) { + rs = false; + } else if ((v1 == null && v2 != null) || (v1 != null && v2 == null)) { + rs = true; + } else if (v1 != null && v2 != null) { + rs = this.compare(v1, v2) != 0; + } + break; + case GT: + rs = this.compare(v1, v2) > 0; + break; + case GE: + rs = this.compare(v1, v2) >= 0; + break; + case LT: + rs = this.compare(v1, v2) < 0; + break; + case LE: + rs = this.compare(v1, v2) <= 0; + break; + case CONTAINS: + if (v1 == null) { + rs = false; + } + if (v1 instanceof Collection && !(v2 instanceof Collection)) { + Collection coll1 = (Collection) v1; + Object el = v2; + if (v2 instanceof Integer || v2 instanceof Long) { + el = Long.valueOf(v2.toString()); + } else if (v2 instanceof Float || v2 instanceof Double) { + el = Double.valueOf(v2.toString()); + } + rs = CollectionUtils.contains(coll1.iterator(), el); + } else if (!(v1 instanceof Collection)) { + throw new FizzRuntimeException("value1 must be a collection"); + } else if (v2 instanceof Collection) { + throw new FizzRuntimeException("value2 can not be a collection"); + } + break; + case NOT_CONTAIN: + if (v1 == null) { + rs = true; + } + if (v1 instanceof Collection && !(v2 instanceof Collection)) { + Collection coll1 = (Collection) v1; + Object el = v2; + if (v2 instanceof Integer || v2 instanceof Long) { + el = Long.valueOf(v2.toString()); + } else if (v2 instanceof Float || v2 instanceof Double) { + el = Double.valueOf(v2.toString()); + } + rs = !CollectionUtils.contains(coll1.iterator(), el); + } else if (!(v1 instanceof Collection)) { + throw new FizzRuntimeException("value1 must be a collection"); + } else if (v2 instanceof Collection) { + throw new FizzRuntimeException("value2 can not be a collection"); + } + break; + case CONTAINS_ANY: + if (v1 == null || v2 == null) { + rs = false; + } + if (v1 instanceof Collection && v2 instanceof Collection) { + Collection coll1 = (Collection) v1; + Collection coll2 = (Collection) v2; + rs = CollectionUtils.containsAny(coll1, coll2); + } else if (!(v1 instanceof Collection)) { + throw new FizzRuntimeException("value1 must be a collection"); + } else if (!(v2 instanceof Collection)) { + throw new FizzRuntimeException("value2 must be a collection"); + } + break; + case IS_NULL: + rs = v1 == null; + break; + case IS_NOT_NULL: + rs = v1 != null; + break; + case IS_BLANK: + rs = v1 == null || StringUtils.isBlank(v1.toString()); + break; + case IS_NOT_BLANK: + rs = v1 != null && StringUtils.isNotBlank(v1.toString()); + break; + case IS_EMPTY: + rs = v1 == null || (v1 instanceof Collection && ((Collection) v1).isEmpty()) + || (v1 instanceof Map && ((Map) v1).isEmpty()); + break; + case IS_NOT_EMPTY: + if (v1 != null) { + if (v1 instanceof Collection) { + rs = !((Collection) v1).isEmpty(); + } else if (v1 instanceof Map) { + rs = !((Map) v1).isEmpty(); + } + } + break; + default: + break; + } + } catch (FizzRuntimeException e) { + String message = type + ": " + e.getMessage() + ", data=" + JSON.toJSONString(this); + throw new FizzRuntimeException(message, e.getCause()); + } + + return rs; + } + + @SuppressWarnings("rawtypes") + private int compare(Object v1, Object v2) { + if (v1 == null || v2 == null) { + throw new FizzRuntimeException("value1 and value2 can not be null"); + } + if (v1 instanceof Boolean && v2 instanceof Boolean) { + Boolean n1 = (Boolean) v1; + Boolean n2 = (Boolean) v2; + return n1.compareTo(n2); + } else if ((v1 instanceof Integer || v1 instanceof Long || v1 instanceof Float || v1 instanceof Double) + && (v2 instanceof Integer || v2 instanceof Long || v2 instanceof Float || v2 instanceof Double)) { + // compare value if both are numbers + Double n1 = Double.valueOf(v1.toString()); + Double n2 = Double.valueOf(v2.toString()); + return n1.compareTo(n2); + } else if (v1 instanceof String && v2 instanceof String) { + String s1 = v1.toString(); + String s2 = v2.toString(); + return s1.compareTo(s2); + } else { + throw new FizzRuntimeException( + "types of value1 and value2 are not consistent or not supported for comparision"); + } + } + + private Object cast(RefDataTypeEnum type, Object val) { + if (type != null) { + switch (type) { + case INT: + val = Integer.valueOf(val.toString()); + break; + case LONG: + val = Long.valueOf(val.toString()); + break; + case FLOAT: + val = Float.valueOf(val.toString()); + break; + case DOUBLE: + val = Double.valueOf(val.toString()); + break; + case BOOLEAN: + val = Boolean.valueOf(val.toString()); + break; + case STRING: + val = val.toString(); + break; + } + } + return val; + } +} diff --git a/fizz-core/src/main/java/we/fizz/component/condition/ConditionValue.java b/fizz-core/src/main/java/we/fizz/component/condition/ConditionValue.java new file mode 100644 index 0000000..ad4cfdb --- /dev/null +++ b/fizz-core/src/main/java/we/fizz/component/condition/ConditionValue.java @@ -0,0 +1,58 @@ +/* + * Copyright (C) 2021 the original author or authors. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package we.fizz.component.condition; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import we.fizz.component.FixedDataTypeEnum; +import we.fizz.component.RefDataTypeEnum; +import we.fizz.component.ValueTypeEnum; + +/** + * Condition value + * + * @author Francis Dong + * + */ +@Data +@AllArgsConstructor +@NoArgsConstructor +public class ConditionValue { + + private ValueTypeEnum type; + + private FixedDataTypeEnum fixedDataType; + + private RefDataTypeEnum refDataType; + + private Object value; + + public ConditionValue(ValueTypeEnum type, FixedDataTypeEnum fixedDataType, Object value) { + this.type = type; + this.fixedDataType = fixedDataType; + this.value = value; + } + + public ConditionValue(ValueTypeEnum type, RefDataTypeEnum refDataType, Object value) { + this.type = type; + this.refDataType = refDataType; + this.value = value; + } + +} diff --git a/fizz-core/src/main/java/we/fizz/exception/FizzRuntimeException.java b/fizz-core/src/main/java/we/fizz/exception/FizzRuntimeException.java index 9205f0b..92643f8 100644 --- a/fizz-core/src/main/java/we/fizz/exception/FizzRuntimeException.java +++ b/fizz-core/src/main/java/we/fizz/exception/FizzRuntimeException.java @@ -1,7 +1,11 @@ package we.fizz.exception; public class FizzRuntimeException extends RuntimeException { - public FizzRuntimeException (String message){ - super(message); - } + public FizzRuntimeException(String message) { + super(message); + } + + public FizzRuntimeException(String message, Throwable cause) { + super(message, cause); + } } diff --git a/fizz-core/src/main/java/we/fizz/input/InputConfig.java b/fizz-core/src/main/java/we/fizz/input/InputConfig.java index 8ce6f91..3e40741 100644 --- a/fizz-core/src/main/java/we/fizz/input/InputConfig.java +++ b/fizz-core/src/main/java/we/fizz/input/InputConfig.java @@ -18,8 +18,11 @@ package we.fizz.input; import java.util.HashMap; +import java.util.List; import java.util.Map; +import we.fizz.component.IComponent; + /** * * @author linwaiwai @@ -30,9 +33,8 @@ public class InputConfig { private InputType type; protected Map dataMapping; protected Map configMap; - - private Map condition; + private List components; public Map getCondition() { return condition; @@ -72,6 +74,14 @@ public class InputConfig { this.fallback = fallback; } + public List getComponents() { + return components; + } + + public void setComponents(List components) { + this.components = components; + } + public void parse(){ } diff --git a/fizz-core/src/main/java/we/fizz/input/InputFactory.java b/fizz-core/src/main/java/we/fizz/input/InputFactory.java index 5a335b7..c0b611b 100644 --- a/fizz-core/src/main/java/we/fizz/input/InputFactory.java +++ b/fizz-core/src/main/java/we/fizz/input/InputFactory.java @@ -17,11 +17,13 @@ package we.fizz.input; +import we.fizz.component.ComponentHelper; import we.fizz.exception.FizzRuntimeException; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.util.HashMap; +import java.util.List; import java.util.Map; /** @@ -56,6 +58,7 @@ public class InputFactory { } inputConfig.setType(typeEnum); inputConfig.setDataMapping((Map) config.get("dataMapping")); + inputConfig.setComponents(ComponentHelper.buildComponents((List>) config.get("components"))); inputConfig.parse(); return inputConfig; } diff --git a/fizz-core/src/main/java/we/fizz/input/PathMapping.java b/fizz-core/src/main/java/we/fizz/input/PathMapping.java index 6f9d46d..2b0c6f8 100644 --- a/fizz-core/src/main/java/we/fizz/input/PathMapping.java +++ b/fizz-core/src/main/java/we/fizz/input/PathMapping.java @@ -309,6 +309,7 @@ public class PathMapping { break; } } + // upper case header name if (list.size() > 5 && "headers".equals(list.get(4))) { String headerName = list.get(5).toUpperCase(); diff --git a/fizz-core/src/main/java/we/fizz/input/extension/dubbo/DubboInput.java b/fizz-core/src/main/java/we/fizz/input/extension/dubbo/DubboInput.java index a5ed012..f34ccef 100644 --- a/fizz-core/src/main/java/we/fizz/input/extension/dubbo/DubboInput.java +++ b/fizz-core/src/main/java/we/fizz/input/extension/dubbo/DubboInput.java @@ -97,7 +97,7 @@ public class DubboInput extends RPCInput { Map group = new HashMap<>(); group.put("request", request); group.put("response", response); - this.stepResponse.getRequests().put(name, group); + this.stepResponse.addRequest(name, group); request.put("serviceName", config.getServiceName()); request.put("version", config.getVersion()); diff --git a/fizz-core/src/main/java/we/fizz/input/extension/grpc/GrpcInput.java b/fizz-core/src/main/java/we/fizz/input/extension/grpc/GrpcInput.java index 0acf716..4829d0d 100644 --- a/fizz-core/src/main/java/we/fizz/input/extension/grpc/GrpcInput.java +++ b/fizz-core/src/main/java/we/fizz/input/extension/grpc/GrpcInput.java @@ -91,7 +91,7 @@ public class GrpcInput extends RPCInput implements IInput { Map group = new HashMap<>(); group.put("request", request); group.put("response", response); - this.stepResponse.getRequests().put(name, group); + this.stepResponse.addRequest(name, group); request.put("serviceName", config.getServiceName()); request.put("method", config.getMethod()); diff --git a/fizz-core/src/main/java/we/fizz/input/extension/request/RequestInput.java b/fizz-core/src/main/java/we/fizz/input/extension/request/RequestInput.java index e0c32f4..d848fd7 100644 --- a/fizz-core/src/main/java/we/fizz/input/extension/request/RequestInput.java +++ b/fizz-core/src/main/java/we/fizz/input/extension/request/RequestInput.java @@ -121,7 +121,7 @@ public class RequestInput extends RPCInput implements IInput{ Map group = new HashMap<>(); group.put("request", request); group.put("response", response); - this.stepResponse.getRequests().put(name, group); + this.stepResponse.addRequest(name, group); HttpMethod method = HttpMethod.valueOf(config.getMethod().toUpperCase()); request.put("method", method); diff --git a/fizz-core/src/test/java/we/fizz/component/CircleTests.java b/fizz-core/src/test/java/we/fizz/component/CircleTests.java new file mode 100644 index 0000000..4e2ee52 --- /dev/null +++ b/fizz-core/src/test/java/we/fizz/component/CircleTests.java @@ -0,0 +1,170 @@ +/* + * Copyright (C) 2021 the original author or authors. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package we.fizz.component; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; + +import org.junit.jupiter.api.Test; +import org.noear.snack.ONode; + +import we.fizz.component.circle.Circle; +import we.fizz.component.circle.CircleItem; +import we.fizz.component.condition.Condition; +import we.fizz.component.condition.ConditionValue; +/** + * + * @author Francis Dong + * + */ +import we.fizz.input.PathMapping; + +class CircleTests { + @Test + void contextLoads() { + } + + + @SuppressWarnings("rawtypes") + @Test + void testNextFixedDataSource() { + ONode ctxNode = ONode.load(new HashMap()); + + // FIXED data source + Circle c = new Circle(null, ValueTypeEnum.FIXED, 3, null, null); + CircleItem circleItem = c.next(ctxNode); + assertEquals(0, (Integer) circleItem.getItem()); + + circleItem = c.next(ctxNode); + assertEquals(1, (Integer) circleItem.getItem()); + + circleItem = c.next(ctxNode); + assertEquals(2, (Integer) circleItem.getItem()); + + circleItem = c.next(ctxNode); + assertEquals(null, circleItem); + + } + + @Test + void testNextRefDataSource() { + ONode ctxNode = ONode.load(new HashMap()); + + List list1 = new ArrayList<>(); + list1.add("1"); + list1.add("2"); + list1.add("3"); + PathMapping.setByPath(ctxNode, "data.list1", list1, true); + + // REF data source + Circle c = new Circle(null, ValueTypeEnum.REF, "data.list1", null, null); + CircleItem circleItem = c.next(ctxNode); + assertEquals("1", (String) circleItem.getItem()); + + circleItem = c.next(ctxNode); + assertEquals("2", (String) circleItem.getItem()); + + circleItem = c.next(ctxNode); + assertEquals("3", (String) circleItem.getItem()); + + circleItem = c.next(ctxNode); + assertEquals(null, circleItem); + + } + + @Test + void testExecCondition() { + ONode ctxNode = ONode.load(new HashMap()); + + List list1 = new ArrayList<>(); + list1.add("0"); + list1.add("1"); + list1.add("2"); + list1.add("3"); + list1.add("4"); + + PathMapping.setByPath(ctxNode, "data.list1", list1, true); + + ConditionValue bValue1 = new ConditionValue(ValueTypeEnum.FIXED, FixedDataTypeEnum.STRING, "3"); + ConditionValue bValue2 = new ConditionValue(ValueTypeEnum.REF, RefDataTypeEnum.STRING, "item"); + Condition c1 = new Condition(null, bValue1, OperatorEnum.NE, bValue2); + + List execConditions = new ArrayList<>(); + execConditions.add(c1); + + Circle circle = new Circle(null, ValueTypeEnum.REF, "data.list1", execConditions, null); + + for (int i = 0; i < 5; i++) { + CircleItem circleItem = circle.next(ctxNode); + PathMapping.setByPath(ctxNode, "item", circleItem.getItem(), true); + PathMapping.setByPath(ctxNode, "index", circleItem.getIndex(), true); + boolean rs = circle.canExec(ctxNode); + assertEquals(i, circleItem.getIndex()); + if (i < 3) { + assertEquals(true, rs); + } + if (i == 3) { + assertEquals(false, rs); + break; + } + } + + } + + @Test + void testBreakCondition() { + ONode ctxNode = ONode.load(new HashMap()); + + List list1 = new ArrayList<>(); + list1.add("0"); + list1.add("1"); + list1.add("2"); + list1.add("3"); + list1.add("4"); + + PathMapping.setByPath(ctxNode, "data.list1", list1, true); + + ConditionValue bValue1 = new ConditionValue(ValueTypeEnum.FIXED, FixedDataTypeEnum.STRING, "3"); + ConditionValue bValue2 = new ConditionValue(ValueTypeEnum.REF, RefDataTypeEnum.STRING, "item"); + Condition c1 = new Condition(null, bValue1, OperatorEnum.EQ, bValue2); + + List breakConditions = new ArrayList<>(); + breakConditions.add(c1); + + Circle circle = new Circle(null, ValueTypeEnum.REF, "data.list1", null, breakConditions); + + for (int i = 0; i < 5; i++) { + CircleItem circleItem = circle.next(ctxNode); + PathMapping.setByPath(ctxNode, "item", circleItem.getItem(), true); + PathMapping.setByPath(ctxNode, "index", circleItem.getIndex(), true); + boolean rs = circle.breakCircle(ctxNode); + assertEquals(i, circleItem.getIndex()); + if (i < 3) { + assertEquals(false, rs); + } + if (i == 3) { + assertEquals(true, rs); + break; + } + } + + } + +} \ No newline at end of file diff --git a/fizz-core/src/test/java/we/fizz/component/ConditionTests.java b/fizz-core/src/test/java/we/fizz/component/ConditionTests.java new file mode 100644 index 0000000..8c478d6 --- /dev/null +++ b/fizz-core/src/test/java/we/fizz/component/ConditionTests.java @@ -0,0 +1,327 @@ +/* + * Copyright (C) 2021 the original author or authors. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package we.fizz.component; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; + +import javax.validation.constraints.AssertTrue; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.noear.snack.ONode; + +import we.fizz.component.condition.Condition; +import we.fizz.component.condition.ConditionValue; +import we.fizz.input.PathMapping; + +/** + * + * @author Francis Dong + * + */ +class ConditionTests { + @Test + void contextLoads() { + } + + private static final Boolean TRUE = true; + private static final Boolean FALSE = false; + + @Test + void testExec() { + ONode ctxNode = ONode.load(new HashMap()); + + Map m = new HashMap<>(); + m.put("int", 1); + m.put("long", 2); + m.put("float", 3.1); + m.put("double", 4.21); + m.put("string_abcd", "abcd"); + m.put("string_1", "1"); + m.put("string_8", "8"); + m.put("string_blank", ""); + m.put("bool_true", true); + m.put("bool_false", false); + + List list1 = new ArrayList<>(); + list1.add("0"); + list1.add("1"); + list1.add("2"); + list1.add("3"); + list1.add("4"); + + List list2 = new ArrayList<>(); + list2.add("1"); + list2.add("3"); + list2.add("223"); + + List intList = new ArrayList<>(); + intList.add(0); + intList.add(1); + intList.add(2); + intList.add(3); + intList.add(4); + + List floatList = new ArrayList<>(); + floatList.add(0f); + floatList.add(1f); + floatList.add(2f); + floatList.add(3f); + floatList.add(4f); + + PathMapping.setByPath(ctxNode, "data.m", m, true); + PathMapping.setByPath(ctxNode, "data.list1", list1, true); + PathMapping.setByPath(ctxNode, "data.list2", list2, true); + PathMapping.setByPath(ctxNode, "data.intList", intList, true); + PathMapping.setByPath(ctxNode, "data.floatList", floatList, true); + PathMapping.setByPath(ctxNode, "data.emptyList", new ArrayList<>(), true); + + // boolean + this.run(ctxNode, new Object[] { ValueTypeEnum.FIXED, FixedDataTypeEnum.BOOLEAN, TRUE, ValueTypeEnum.FIXED, + FixedDataTypeEnum.BOOLEAN, TRUE, OperatorEnum.EQ, TRUE }); + this.run(ctxNode, new Object[] { ValueTypeEnum.FIXED, FixedDataTypeEnum.BOOLEAN, FALSE, ValueTypeEnum.FIXED, + FixedDataTypeEnum.BOOLEAN, FALSE, OperatorEnum.EQ, TRUE }); + this.run(ctxNode, new Object[] { ValueTypeEnum.FIXED, FixedDataTypeEnum.BOOLEAN, TRUE, ValueTypeEnum.FIXED, + FixedDataTypeEnum.BOOLEAN, FALSE, OperatorEnum.EQ, FALSE }); + + this.run(ctxNode, new Object[] { ValueTypeEnum.FIXED, FixedDataTypeEnum.BOOLEAN, FALSE, ValueTypeEnum.REF, + RefDataTypeEnum.BOOLEAN, "data.m.bool_true", OperatorEnum.EQ, FALSE }); + this.run(ctxNode, new Object[] { ValueTypeEnum.FIXED, FixedDataTypeEnum.BOOLEAN, FALSE, ValueTypeEnum.REF, + RefDataTypeEnum.BOOLEAN, "data.m.bool_false", OperatorEnum.EQ, TRUE }); + this.run(ctxNode, new Object[] { ValueTypeEnum.FIXED, FixedDataTypeEnum.BOOLEAN, TRUE, ValueTypeEnum.REF, + RefDataTypeEnum.BOOLEAN, "data.m.bool_true", OperatorEnum.EQ, TRUE }); + this.run(ctxNode, new Object[] { ValueTypeEnum.FIXED, FixedDataTypeEnum.BOOLEAN, FALSE, ValueTypeEnum.REF, null, + "data.m.a", OperatorEnum.EQ, FALSE }); + + this.run(ctxNode, new Object[] { ValueTypeEnum.FIXED, FixedDataTypeEnum.BOOLEAN, TRUE, ValueTypeEnum.FIXED, + FixedDataTypeEnum.BOOLEAN, FALSE, OperatorEnum.GT, TRUE }); + this.run(ctxNode, new Object[] { ValueTypeEnum.FIXED, FixedDataTypeEnum.BOOLEAN, TRUE, ValueTypeEnum.FIXED, + FixedDataTypeEnum.BOOLEAN, FALSE, OperatorEnum.GE, TRUE }); + this.run(ctxNode, new Object[] { ValueTypeEnum.FIXED, FixedDataTypeEnum.BOOLEAN, TRUE, ValueTypeEnum.FIXED, + FixedDataTypeEnum.BOOLEAN, TRUE, OperatorEnum.GE, TRUE }); + this.run(ctxNode, new Object[] { ValueTypeEnum.FIXED, FixedDataTypeEnum.BOOLEAN, FALSE, ValueTypeEnum.FIXED, + FixedDataTypeEnum.BOOLEAN, TRUE, OperatorEnum.LT, TRUE }); + this.run(ctxNode, new Object[] { ValueTypeEnum.FIXED, FixedDataTypeEnum.BOOLEAN, FALSE, ValueTypeEnum.FIXED, + FixedDataTypeEnum.BOOLEAN, FALSE, OperatorEnum.LE, TRUE }); + + // number + this.run(ctxNode, new Object[] { ValueTypeEnum.FIXED, FixedDataTypeEnum.NUMBER, 1, ValueTypeEnum.FIXED, + FixedDataTypeEnum.NUMBER, 1, OperatorEnum.EQ, TRUE }); + this.run(ctxNode, new Object[] { ValueTypeEnum.FIXED, FixedDataTypeEnum.NUMBER, 1, ValueTypeEnum.FIXED, + FixedDataTypeEnum.NUMBER, 2, OperatorEnum.EQ, FALSE }); + this.run(ctxNode, new Object[] { ValueTypeEnum.FIXED, FixedDataTypeEnum.NUMBER, 1, ValueTypeEnum.FIXED, + FixedDataTypeEnum.NUMBER, 1.000, OperatorEnum.EQ, TRUE }); + this.run(ctxNode, new Object[] { ValueTypeEnum.FIXED, FixedDataTypeEnum.NUMBER, 1, ValueTypeEnum.FIXED, + FixedDataTypeEnum.NUMBER, 2.1, OperatorEnum.EQ, FALSE }); + this.run(ctxNode, new Object[] { ValueTypeEnum.FIXED, FixedDataTypeEnum.NUMBER, 1.0, ValueTypeEnum.FIXED, + FixedDataTypeEnum.NUMBER, 1.000, OperatorEnum.EQ, TRUE }); + this.run(ctxNode, new Object[] { ValueTypeEnum.FIXED, FixedDataTypeEnum.NUMBER, 1.1, ValueTypeEnum.FIXED, + FixedDataTypeEnum.NUMBER, 2.1, OperatorEnum.EQ, FALSE }); + + this.run(ctxNode, new Object[] { ValueTypeEnum.FIXED, FixedDataTypeEnum.NUMBER, 1.1, ValueTypeEnum.FIXED, + FixedDataTypeEnum.NUMBER, 2.1, OperatorEnum.GT, FALSE }); + this.run(ctxNode, new Object[] { ValueTypeEnum.FIXED, FixedDataTypeEnum.NUMBER, 1.1, ValueTypeEnum.FIXED, + FixedDataTypeEnum.NUMBER, 0.1, OperatorEnum.GE, TRUE }); + this.run(ctxNode, new Object[] { ValueTypeEnum.FIXED, FixedDataTypeEnum.NUMBER, 1.1, ValueTypeEnum.FIXED, + FixedDataTypeEnum.NUMBER, 0.1, OperatorEnum.LT, FALSE }); + this.run(ctxNode, new Object[] { ValueTypeEnum.FIXED, FixedDataTypeEnum.NUMBER, 1.1, ValueTypeEnum.FIXED, + FixedDataTypeEnum.NUMBER, 4, OperatorEnum.LT, TRUE }); + + // collection + this.run(ctxNode, new Object[] { ValueTypeEnum.REF, RefDataTypeEnum.ARRAY, "data.list1", ValueTypeEnum.FIXED, + FixedDataTypeEnum.STRING, "2", OperatorEnum.CONTAINS, TRUE }); + this.run(ctxNode, new Object[] { ValueTypeEnum.REF, RefDataTypeEnum.ARRAY, "data.list1", ValueTypeEnum.REF, + RefDataTypeEnum.STRING, "data.m.string_1", OperatorEnum.CONTAINS, TRUE }); + this.run(ctxNode, new Object[] { ValueTypeEnum.REF, RefDataTypeEnum.ARRAY, "data.list1", ValueTypeEnum.REF, + RefDataTypeEnum.STRING, "data.m.string_8", OperatorEnum.CONTAINS, FALSE }); + + this.run(ctxNode, new Object[] { ValueTypeEnum.REF, RefDataTypeEnum.ARRAY, "data.list1", ValueTypeEnum.FIXED, + FixedDataTypeEnum.STRING, "2", OperatorEnum.NOT_CONTAIN, FALSE }); + this.run(ctxNode, new Object[] { ValueTypeEnum.REF, RefDataTypeEnum.ARRAY, "data.list1", ValueTypeEnum.REF, + RefDataTypeEnum.STRING, "data.m.string_1", OperatorEnum.NOT_CONTAIN, FALSE }); + this.run(ctxNode, new Object[] { ValueTypeEnum.REF, RefDataTypeEnum.ARRAY, "data.list1", ValueTypeEnum.REF, + RefDataTypeEnum.STRING, "data.m.string_8", OperatorEnum.NOT_CONTAIN, TRUE }); + + // collection contains any + this.run(ctxNode, new Object[] { ValueTypeEnum.REF, RefDataTypeEnum.ARRAY, "data.list1", ValueTypeEnum.REF, + RefDataTypeEnum.ARRAY, "data.list2", OperatorEnum.CONTAINS_ANY, TRUE }); + this.run(ctxNode, new Object[] { ValueTypeEnum.REF, RefDataTypeEnum.ARRAY, "data.list1", ValueTypeEnum.REF, + RefDataTypeEnum.ARRAY, "data.intList", OperatorEnum.CONTAINS_ANY, FALSE }); + + // Collection + this.run(ctxNode, new Object[] { ValueTypeEnum.REF, RefDataTypeEnum.ARRAY, "data.intList", ValueTypeEnum.FIXED, + FixedDataTypeEnum.STRING, "2", OperatorEnum.CONTAINS, FALSE }); + this.run(ctxNode, new Object[] { ValueTypeEnum.REF, RefDataTypeEnum.ARRAY, "data.intList", ValueTypeEnum.REF, + RefDataTypeEnum.INT, "data.m.int", OperatorEnum.CONTAINS, TRUE }); + this.run(ctxNode, new Object[] { ValueTypeEnum.REF, RefDataTypeEnum.ARRAY, "data.intList", ValueTypeEnum.FIXED, + RefDataTypeEnum.INT, 2, OperatorEnum.CONTAINS, TRUE }); + this.run(ctxNode, new Object[] { ValueTypeEnum.REF, RefDataTypeEnum.ARRAY, "data.intList", ValueTypeEnum.FIXED, + RefDataTypeEnum.INT, 9, OperatorEnum.CONTAINS, FALSE }); + + this.run(ctxNode, new Object[] { ValueTypeEnum.REF, RefDataTypeEnum.ARRAY, "data.intList", ValueTypeEnum.FIXED, + FixedDataTypeEnum.STRING, "2", OperatorEnum.NOT_CONTAIN, TRUE }); + this.run(ctxNode, new Object[] { ValueTypeEnum.REF, RefDataTypeEnum.ARRAY, "data.intList", ValueTypeEnum.REF, + RefDataTypeEnum.INT, "data.m.int", OperatorEnum.NOT_CONTAIN, FALSE }); + this.run(ctxNode, new Object[] { ValueTypeEnum.REF, RefDataTypeEnum.ARRAY, "data.intList", ValueTypeEnum.FIXED, + RefDataTypeEnum.INT, 2, OperatorEnum.NOT_CONTAIN, FALSE }); + this.run(ctxNode, new Object[] { ValueTypeEnum.REF, RefDataTypeEnum.ARRAY, "data.intList", ValueTypeEnum.FIXED, + RefDataTypeEnum.INT, 9, OperatorEnum.NOT_CONTAIN, TRUE }); + + // Collection + this.run(ctxNode, new Object[] { ValueTypeEnum.REF, RefDataTypeEnum.ARRAY, "data.floatList", + ValueTypeEnum.FIXED, FixedDataTypeEnum.STRING, "2", OperatorEnum.CONTAINS, FALSE }); + this.run(ctxNode, new Object[] { ValueTypeEnum.REF, RefDataTypeEnum.ARRAY, "data.floatList", ValueTypeEnum.REF, + RefDataTypeEnum.INT, "data.m.int", OperatorEnum.CONTAINS, FALSE }); + this.run(ctxNode, new Object[] { ValueTypeEnum.REF, RefDataTypeEnum.ARRAY, "data.floatList", + ValueTypeEnum.FIXED, FixedDataTypeEnum.NUMBER, 2, OperatorEnum.CONTAINS, FALSE }); + this.run(ctxNode, new Object[] { ValueTypeEnum.REF, RefDataTypeEnum.ARRAY, "data.floatList", + ValueTypeEnum.FIXED, FixedDataTypeEnum.NUMBER, 2.0, OperatorEnum.CONTAINS, TRUE }); + + // String + this.run(ctxNode, new Object[] { ValueTypeEnum.FIXED, FixedDataTypeEnum.STRING, null, ValueTypeEnum.FIXED, + FixedDataTypeEnum.STRING, null, OperatorEnum.EQ, TRUE }); + this.run(ctxNode, new Object[] { ValueTypeEnum.FIXED, FixedDataTypeEnum.STRING, null, ValueTypeEnum.FIXED, + FixedDataTypeEnum.STRING, null, OperatorEnum.EQ, TRUE }); + this.run(ctxNode, new Object[] { ValueTypeEnum.FIXED, FixedDataTypeEnum.STRING, null, ValueTypeEnum.FIXED, + FixedDataTypeEnum.STRING, "1", OperatorEnum.EQ, FALSE }); + this.run(ctxNode, new Object[] { ValueTypeEnum.FIXED, FixedDataTypeEnum.STRING, "1", ValueTypeEnum.FIXED, + FixedDataTypeEnum.STRING, "1", OperatorEnum.EQ, TRUE }); + this.run(ctxNode, new Object[] { ValueTypeEnum.FIXED, FixedDataTypeEnum.STRING, "1", ValueTypeEnum.FIXED, + FixedDataTypeEnum.STRING, "21", OperatorEnum.EQ, FALSE }); + + this.run(ctxNode, new Object[] { ValueTypeEnum.FIXED, FixedDataTypeEnum.STRING, null, ValueTypeEnum.FIXED, + FixedDataTypeEnum.STRING, null, OperatorEnum.NE, FALSE }); + this.run(ctxNode, new Object[] { ValueTypeEnum.FIXED, FixedDataTypeEnum.STRING, null, ValueTypeEnum.FIXED, + FixedDataTypeEnum.STRING, null, OperatorEnum.NE, FALSE }); + this.run(ctxNode, new Object[] { ValueTypeEnum.FIXED, FixedDataTypeEnum.STRING, null, ValueTypeEnum.FIXED, + FixedDataTypeEnum.STRING, "1", OperatorEnum.NE, TRUE }); + this.run(ctxNode, new Object[] { ValueTypeEnum.FIXED, FixedDataTypeEnum.STRING, "1", ValueTypeEnum.FIXED, + FixedDataTypeEnum.STRING, "1", OperatorEnum.NE, FALSE }); + this.run(ctxNode, new Object[] { ValueTypeEnum.FIXED, FixedDataTypeEnum.STRING, "1", ValueTypeEnum.FIXED, + FixedDataTypeEnum.STRING, "21", OperatorEnum.NE, TRUE }); + + this.run(ctxNode, new Object[] { ValueTypeEnum.FIXED, FixedDataTypeEnum.STRING, "1", ValueTypeEnum.FIXED, + FixedDataTypeEnum.STRING, "21", OperatorEnum.GT, FALSE }); + this.run(ctxNode, new Object[] { ValueTypeEnum.FIXED, FixedDataTypeEnum.STRING, "11", ValueTypeEnum.FIXED, + FixedDataTypeEnum.STRING, "21", OperatorEnum.GT, FALSE }); + this.run(ctxNode, new Object[] { ValueTypeEnum.FIXED, FixedDataTypeEnum.STRING, "11", ValueTypeEnum.FIXED, + FixedDataTypeEnum.STRING, "21", OperatorEnum.GE, FALSE }); + this.run(ctxNode, new Object[] { ValueTypeEnum.FIXED, FixedDataTypeEnum.STRING, "11", ValueTypeEnum.FIXED, + FixedDataTypeEnum.STRING, "21", OperatorEnum.LT, TRUE }); + this.run(ctxNode, new Object[] { ValueTypeEnum.FIXED, FixedDataTypeEnum.STRING, "11", ValueTypeEnum.FIXED, + FixedDataTypeEnum.STRING, "21", OperatorEnum.LE, TRUE }); + + // Is null + this.run(ctxNode, new Object[] { ValueTypeEnum.REF, RefDataTypeEnum.ARRAY, "data.list11111", ValueTypeEnum.REF, + RefDataTypeEnum.STRING, "data.m.string_8", OperatorEnum.IS_NULL, TRUE }); + this.run(ctxNode, new Object[] { ValueTypeEnum.REF, RefDataTypeEnum.STRING, "data.m.string_8", + ValueTypeEnum.REF, RefDataTypeEnum.STRING, "data.m.string_8", OperatorEnum.IS_NULL, FALSE }); + this.run(ctxNode, new Object[] { ValueTypeEnum.REF, RefDataTypeEnum.ARRAY, "data.list11111", null, null, null, + OperatorEnum.IS_NULL, TRUE }); + this.run(ctxNode, new Object[] { ValueTypeEnum.REF, RefDataTypeEnum.STRING, "data.m.string_8", null, null, null, + OperatorEnum.IS_NULL, FALSE }); + + // Is not null + this.run(ctxNode, new Object[] { ValueTypeEnum.REF, RefDataTypeEnum.ARRAY, "data.list11111", ValueTypeEnum.REF, + RefDataTypeEnum.STRING, "data.m.string_8", OperatorEnum.IS_NOT_NULL, FALSE }); + this.run(ctxNode, new Object[] { ValueTypeEnum.REF, RefDataTypeEnum.STRING, "data.m.string_8", + ValueTypeEnum.REF, RefDataTypeEnum.STRING, "data.m.string_8", OperatorEnum.IS_NOT_NULL, TRUE }); + this.run(ctxNode, new Object[] { ValueTypeEnum.REF, RefDataTypeEnum.ARRAY, "data.list11111", null, null, null, + OperatorEnum.IS_NOT_NULL, FALSE }); + this.run(ctxNode, new Object[] { ValueTypeEnum.REF, RefDataTypeEnum.STRING, "data.m.string_8", null, null, null, + OperatorEnum.IS_NOT_NULL, TRUE }); + + // Is Blank + this.run(ctxNode, new Object[] { ValueTypeEnum.REF, RefDataTypeEnum.ARRAY, "data.list11111", ValueTypeEnum.REF, + RefDataTypeEnum.STRING, "data.m.string_8", OperatorEnum.IS_BLANK, TRUE }); + this.run(ctxNode, new Object[] { ValueTypeEnum.REF, RefDataTypeEnum.STRING, "data.m.string_8", + ValueTypeEnum.REF, RefDataTypeEnum.STRING, "data.m.string_8", OperatorEnum.IS_BLANK, FALSE }); + this.run(ctxNode, new Object[] { ValueTypeEnum.REF, RefDataTypeEnum.ARRAY, "data.list11111", null, null, null, + OperatorEnum.IS_BLANK, TRUE }); + this.run(ctxNode, new Object[] { ValueTypeEnum.REF, RefDataTypeEnum.STRING, "data.m.string_8", null, null, null, + OperatorEnum.IS_BLANK, FALSE }); + this.run(ctxNode, new Object[] { ValueTypeEnum.REF, RefDataTypeEnum.STRING, "data.m.string_blank", null, null, + null, OperatorEnum.IS_BLANK, TRUE }); + + // Is not Blank + this.run(ctxNode, new Object[] { ValueTypeEnum.REF, RefDataTypeEnum.ARRAY, "data.list11111", ValueTypeEnum.REF, + RefDataTypeEnum.STRING, "data.m.string_8", OperatorEnum.IS_NOT_BLANK, FALSE }); + this.run(ctxNode, new Object[] { ValueTypeEnum.REF, RefDataTypeEnum.STRING, "data.m.string_8", + ValueTypeEnum.REF, RefDataTypeEnum.STRING, "data.m.string_8", OperatorEnum.IS_NOT_BLANK, TRUE }); + this.run(ctxNode, new Object[] { ValueTypeEnum.REF, RefDataTypeEnum.ARRAY, "data.list11111", null, null, null, + OperatorEnum.IS_NOT_BLANK, FALSE }); + this.run(ctxNode, new Object[] { ValueTypeEnum.REF, RefDataTypeEnum.STRING, "data.m.string_8", null, null, null, + OperatorEnum.IS_NOT_BLANK, TRUE }); + this.run(ctxNode, new Object[] { ValueTypeEnum.REF, RefDataTypeEnum.STRING, "data.m.string_blank", null, null, + null, OperatorEnum.IS_NOT_BLANK, FALSE }); + + // Is empty + this.run(ctxNode, new Object[] { ValueTypeEnum.REF, RefDataTypeEnum.ARRAY, "data.list11111", ValueTypeEnum.REF, + RefDataTypeEnum.STRING, "data.m.string_8", OperatorEnum.IS_EMPTY, TRUE }); + this.run(ctxNode, new Object[] { ValueTypeEnum.REF, RefDataTypeEnum.ARRAY, "data.list1", ValueTypeEnum.REF, + RefDataTypeEnum.STRING, "data.m.string_8", OperatorEnum.IS_EMPTY, FALSE }); + this.run(ctxNode, new Object[] { ValueTypeEnum.REF, RefDataTypeEnum.ARRAY, "data.list11111", null, null, null, + OperatorEnum.IS_EMPTY, TRUE }); + this.run(ctxNode, new Object[] { ValueTypeEnum.REF, RefDataTypeEnum.ARRAY, "data.list1", null, null, null, + OperatorEnum.IS_EMPTY, FALSE }); + this.run(ctxNode, new Object[] { ValueTypeEnum.REF, RefDataTypeEnum.ARRAY, "data.emptyList", null, null, null, + OperatorEnum.IS_EMPTY, TRUE }); + + // Is not empty + this.run(ctxNode, new Object[] { ValueTypeEnum.REF, RefDataTypeEnum.ARRAY, "data.list11111", ValueTypeEnum.REF, + RefDataTypeEnum.STRING, "data.m.string_8", OperatorEnum.IS_NOT_EMPTY, FALSE }); + this.run(ctxNode, new Object[] { ValueTypeEnum.REF, RefDataTypeEnum.ARRAY, "data.list1", ValueTypeEnum.REF, + RefDataTypeEnum.STRING, "data.m.string_8", OperatorEnum.IS_NOT_EMPTY, TRUE }); + this.run(ctxNode, new Object[] { ValueTypeEnum.REF, RefDataTypeEnum.ARRAY, "data.list11111", null, null, null, + OperatorEnum.IS_NOT_EMPTY, FALSE }); + this.run(ctxNode, new Object[] { ValueTypeEnum.REF, RefDataTypeEnum.ARRAY, "data.list1", null, null, null, + OperatorEnum.IS_NOT_EMPTY, TRUE }); + this.run(ctxNode, new Object[] { ValueTypeEnum.REF, RefDataTypeEnum.ARRAY, "data.emptyList", null, null, null, + OperatorEnum.IS_NOT_EMPTY, FALSE }); + + } + + private void run(ONode ctxNode, Object[] item) { + ConditionValue bValue1 = null; + if (item[1] instanceof FixedDataTypeEnum) { + bValue1 = new ConditionValue((ValueTypeEnum) item[0], (FixedDataTypeEnum) item[1], item[2]); + } else { + bValue1 = new ConditionValue((ValueTypeEnum) item[0], (RefDataTypeEnum) item[1], item[2]); + } + ConditionValue bValue2 = null; + if (item[3] != null) { + if (item[4] instanceof FixedDataTypeEnum) { + bValue2 = new ConditionValue((ValueTypeEnum) item[3], (FixedDataTypeEnum) item[4], item[5]); + } else { + bValue2 = new ConditionValue((ValueTypeEnum) item[3], (RefDataTypeEnum) item[4], item[5]); + } + } + Condition c = new Condition(null, bValue1, (OperatorEnum) item[6], bValue2); + boolean rs = c.exec(ctxNode); + boolean expected = (boolean) item[7]; + assertEquals(expected, rs); + } + +} \ No newline at end of file diff --git a/fizz-core/src/test/java/we/fizz/input/PathMappingTests.java b/fizz-core/src/test/java/we/fizz/input/PathMappingTests.java index 770ba26..4fd6475 100644 --- a/fizz-core/src/test/java/we/fizz/input/PathMappingTests.java +++ b/fizz-core/src/test/java/we/fizz/input/PathMappingTests.java @@ -163,4 +163,31 @@ class PathMappingTests { } + @Test + void testArray() { + ONode ctxNode = ONode.load(new HashMap()); + + Map m = new HashMap<>(); + m.put("a", "1"); + m.put("b", "1"); + + List list = new ArrayList<>(); + list.add("0"); + list.add("1"); + list.add("2"); + list.add("3"); + list.add("4"); + + PathMapping.setByPath(ctxNode, "data.m", m, true); + + PathMapping.setByPath(ctxNode, "data.arr", list, true); + + Object abcVal1 = PathMapping.getValueByPath(ctxNode, "data.arr[0]"); + assertEquals("0", (String)abcVal1); + Object abcVal2 = PathMapping.getValueByPath(ctxNode, "data.arr[-1]"); + assertEquals("4", (String)abcVal2); + System.out.println(abcVal1); + System.out.println(abcVal2); + } + } \ No newline at end of file From bada9ceea12b91eac5acf012a60090aa3f1d7e4a Mon Sep 17 00:00:00 2001 From: Francis Dong Date: Thu, 8 Jul 2021 12:01:27 +0800 Subject: [PATCH 25/25] Support default value in jsonpath mapping #216 --- .../main/java/we/fizz/input/PathMapping.java | 56 +++++++++++++++---- .../java/we/fizz/input/PathMappingTests.java | 4 +- 2 files changed, 48 insertions(+), 12 deletions(-) diff --git a/fizz-core/src/main/java/we/fizz/input/PathMapping.java b/fizz-core/src/main/java/we/fizz/input/PathMapping.java index 2b0c6f8..91b1a92 100644 --- a/fizz-core/src/main/java/we/fizz/input/PathMapping.java +++ b/fizz-core/src/main/java/we/fizz/input/PathMapping.java @@ -135,39 +135,73 @@ public class PathMapping { String starEntryKey = null; for (Entry entry : rs.entrySet()) { - String path = handlePath(entry.getValue()); - ONode val = select(ctxNode, path); - Object obj = val; - if (val != null && types.containsKey(entry.getKey())) { + String path = entry.getValue(); + String p = path; + String defaultValue = null; + if (path.indexOf("|") != -1) { + p = path.substring(0, path.indexOf("|")); + defaultValue = path.substring(path.indexOf("|") + 1); + } + ONode val = select(ctxNode, handlePath(p)); + + Object obj = null; + if (val != null && !val.isNull()) { + obj = val; + } else { + obj = defaultValue; + } + if (obj != null && types.containsKey(entry.getKey())) { switch (types.get(entry.getKey())) { case "Integer": case "int": { - obj = val.val().isNull() ? null : val.val().getInt(); + if (obj instanceof ONode) { + obj = ((ONode) obj).val().getInt(); + } else { + obj = Integer.valueOf(obj.toString()); + } break; } case "Boolean": case "boolean": { - obj = val.val().isNull() ? null : val.val().getBoolean(); + if (obj instanceof ONode) { + obj = ((ONode) obj).val().getBoolean(); + } else { + obj = Boolean.valueOf(obj.toString()); + } break; } case "Float": case "float": { - obj = val.val().isNull() ? null : val.val().getFloat(); + if (obj instanceof ONode) { + obj = ((ONode) obj).val().getFloat(); + } else { + obj = Float.valueOf(obj.toString()); + } break; } case "Double": case "double": { - obj = val.val().isNull() ? null : val.val().getDouble(); + if (obj instanceof ONode) { + obj = ((ONode) obj).val().getDouble(); + } else { + obj = Double.valueOf(obj.toString()); + } break; } case "String": case "string": { - obj = val.val().isNull() ? null : val.val().getString(); + if (obj instanceof ONode) { + obj = ((ONode) obj).val().getString(); + } break; } case "Long": case "long": { - obj = val.val().isNull() ? null : val.val().getLong(); + if (obj instanceof ONode) { + obj = ((ONode) obj).val().getLong(); + } else { + obj = Long.valueOf(obj.toString()); + } break; } } @@ -175,7 +209,7 @@ public class PathMapping { if (CommonConstants.WILDCARD_STAR.equals(entry.getKey())) { starValObj = obj; starEntryKey = entry.getKey(); - }else { + } else { setByPath(target, entry.getKey(), obj, supportMultiLevels); } } diff --git a/fizz-core/src/test/java/we/fizz/input/PathMappingTests.java b/fizz-core/src/test/java/we/fizz/input/PathMappingTests.java index 4fd6475..4936d2b 100644 --- a/fizz-core/src/test/java/we/fizz/input/PathMappingTests.java +++ b/fizz-core/src/test/java/we/fizz/input/PathMappingTests.java @@ -14,6 +14,9 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.noear.snack.ONode; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + class PathMappingTests { @Test @@ -159,7 +162,6 @@ class PathMappingTests { assertEquals("1", (String)abcVal1); assertEquals("123456", (String)abcVal2); - }