fizzgate/fizz-gateway-node
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

optimization1

Browse Source
This commit is contained in:
hongqiaowei
2022-07-09 11:07:15 +08:00
parent 825959e5d6
commit 7799360a80
6 changed files with 234 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
fizz-bootstrap/pom.xml
View File

@@ -56,6 +56,13 @@
<artifactId>fizz-spring-boot-starter</artifactId> <artifactId>fizz-spring-boot-starter</artifactId>
<version>${project.version}</version> <version>${project.version}</version>
</dependency> </dependency>
<dependency>
<groupId>com.networknt</groupId>
<artifactId>json-schema-validator-i18n-support</artifactId>
<version>1.0.39_5</version>
</dependency>
<!-- import fizz-input-mysql --> <!-- import fizz-input-mysql -->
<!-- <dependency> <!-- <dependency>
<groupId>com.fizzgate</groupId> <groupId>com.fizzgate</groupId>
@@ -111,6 +118,13 @@
</profile>--> </profile>-->
</profiles> </profiles>
<repositories>
<repository>
<id>repo</id>
<url>file://${project.basedir}/../repo</url>
</repository>
</repositories>
<build> <build>
<plugins> <plugins>
<plugin> <plugin>
@@ -118,6 +132,12 @@
<artifactId>spring-boot-maven-plugin</artifactId> <artifactId>spring-boot-maven-plugin</artifactId>
<configuration> <configuration>
<includeSystemScope>true</includeSystemScope> <includeSystemScope>true</includeSystemScope>
<excludes>
<exclude>
<groupId>com.networknt</groupId>
<artifactId>json-schema-validator-i18n-support</artifactId>
</exclude>
</excludes>
</configuration> </configuration>
</plugin> </plugin>
<plugin> <plugin>

30
fizz-common/src/main/java/we/util/DateTimeUtils.java
View File

@@ -17,7 +17,12 @@
package we.util; package we.util;
import java.time.*; import we.util.Consts.DP;
import java.time.Instant;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter; import java.time.format.DateTimeFormatter;
import java.time.temporal.ChronoField; import java.time.temporal.ChronoField;
import java.time.temporal.ChronoUnit; import java.time.temporal.ChronoUnit;
@@ -26,8 +31,6 @@ import java.util.stream.Collectors;
import java.util.stream.IntStream; import java.util.stream.IntStream;
import java.util.stream.Stream; import java.util.stream.Stream;
import we.util.Consts.DP;
/** /**
* @author hongqiaowei * @author hongqiaowei
*/ */
@@ -165,6 +168,27 @@ public abstract class DateTimeUtils {
return localDate1.isEqual(localDate2); return localDate1.isEqual(localDate2);
} }
public static long get10sTimeWinStart(int n) {
LocalDateTime now = LocalDateTime.now().with(ChronoField.MILLI_OF_SECOND, 0);
int sec = now.getSecond();
long interval;
if (sec > 49) {
interval = sec - 50;
} else if (sec > 39) {
interval = sec - 40;
} else if (sec > 29) {
interval = sec - 30;
} else if (sec > 19) {
interval = sec - 20;
} else if (sec > 9) {
interval = sec - 10;
} else {
interval = sec;
}
long millis = toMillis(now);
return millis - interval * 1000 - (n - 1) * 10L * 1000;
}
/* /*
void iterateBetweenDatesJava8(LocalDate start, LocalDate end) { void iterateBetweenDatesJava8(LocalDate start, LocalDate end) {
for (LocalDate date = start; date.isBefore(end); date = date.plusDays(1)) { for (LocalDate date = start; date.isBefore(end); date = date.plusDays(1)) {

39
fizz-core/src/main/java/we/plugin/stat/AccessStat.java Normal file
View File

@@ -0,0 +1,39 @@
/*
* Copyright (C) 2021 the original author or authors.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package we.plugin.stat;
import we.util.JacksonUtils;
/**
* @author hongqiaowei
*/
public class AccessStat {
public String service;
public String apiMethod;
public String apiPath;
public long start;
public int reqs = 0;
public long reqTime;
@Override
public String toString() {
return JacksonUtils.writeValueAsString(this);
}
}

84
fizz-core/src/main/java/we/plugin/stat/AccessStatSchedConfig.java Normal file
View File

@@ -0,0 +1,84 @@
/*
* Copyright (C) 2021 the original author or authors.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package we.plugin.stat;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.context.annotation.Configuration;
import org.springframework.data.redis.core.ReactiveStringRedisTemplate;
import org.springframework.scheduling.annotation.Scheduled;
import we.config.AggregateRedisConfig;
import we.config.SchedConfig;
import we.util.Consts;
import we.util.DateTimeUtils;
import we.util.StringUtils;
import javax.annotation.Resource;
import java.util.Map;
/**
* @author hongqiaowei
*/
@Configuration
public class AccessStatSchedConfig extends SchedConfig {
private static final Logger LOGGER = LoggerFactory.getLogger(AccessStatSchedConfig.class);
private static final Logger STAT_LOGGER = LoggerFactory.getLogger("stat");
@Resource(name = AggregateRedisConfig.AGGREGATE_REACTIVE_REDIS_TEMPLATE)
private ReactiveStringRedisTemplate rt;
@Resource
private StatPluginFilterProperties statPluginFilterProperties;
@Resource
private StatPluginFilter statPluginFilter;
@Scheduled(cron = "${fizz-access-stat-sched.cron:2/10 * * * * ?}")
public void sched() {
long prevTimeWinStart = DateTimeUtils.get10sTimeWinStart(2);
Map<String, AccessStat> accessStatMap = statPluginFilter.getAccessStat(prevTimeWinStart);
if (accessStatMap.isEmpty()) {
if (LOGGER.isDebugEnabled()) {
LOGGER.debug("no access stat in {} window", DateTimeUtils.convert(prevTimeWinStart, Consts.DP.DP19));
}
} else {
accessStatMap.forEach(
(smp, accessStat) -> {
String msg = accessStat.toString();
String topic = statPluginFilterProperties.getFizzAccessStatTopic();
if (StringUtils.isBlank(topic)) {
String channel = statPluginFilterProperties.getFizzAccessStatChannel();
rt.convertAndSend(channel, msg).subscribe();
if (LOGGER.isDebugEnabled()) {
LOGGER.debug("send access stat {} which belong to {} window to channel {}", msg, DateTimeUtils.convert(accessStat.start, Consts.DP.DP19), channel);
}
} else {
STAT_LOGGER.info(msg);
if (LOGGER.isDebugEnabled()) {
LOGGER.debug("send access stat {} which belong to {} window to topic", msg, DateTimeUtils.convert(accessStat.start, Consts.DP.DP19));
}
}
}
);
}
}
}

125
fizz-core/src/main/java/we/plugin/stat/StatPluginFilter.java
View File

@@ -17,21 +17,20 @@
package we.plugin.stat; package we.plugin.stat;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.springframework.data.redis.core.ReactiveStringRedisTemplate;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import org.springframework.web.server.ServerWebExchange; import org.springframework.web.server.ServerWebExchange;
import reactor.core.publisher.Mono; import reactor.core.publisher.Mono;
import we.config.AggregateRedisConfig;
import we.plugin.PluginFilter; import we.plugin.PluginFilter;
import we.plugin.auth.GatewayGroupService;
import we.util.Consts; import we.util.Consts;
import we.util.DateTimeUtils;
import we.util.ThreadContext; import we.util.ThreadContext;
import we.util.WebUtils; import we.util.WebUtils;
import javax.annotation.Resource; import javax.annotation.Resource;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.Map; import java.util.Map;
/** /**
@@ -42,82 +41,84 @@ import java.util.Map;
@Component(StatPluginFilter.STAT_PLUGIN_FILTER) @Component(StatPluginFilter.STAT_PLUGIN_FILTER)
public class StatPluginFilter extends PluginFilter { public class StatPluginFilter extends PluginFilter {
private static final Logger log = LoggerFactory.getLogger("stat"); private static final Logger LOGGER = LoggerFactory.getLogger(StatPluginFilter.class);
public static final String STAT_PLUGIN_FILTER = "statPlugin"; public static final String STAT_PLUGIN_FILTER = "statPlugin";
private static final String ip = "\"ip\":";
private static final String gatewayGroup = "\"gatewayGroup\":";
private static final String service = "\"service\":";
private static final String appid = "\"appid\":";
private static final String apiMethod = "\"apiMethod\":";
private static final String apiPath = "\"apiPath\":";
private static final String reqTime = "\"reqTime\":";
@Resource @Resource
private StatPluginFilterProperties statPluginFilterProperties; private StatPluginFilterProperties statPluginFilterProperties;
@Resource(name = AggregateRedisConfig.AGGREGATE_REACTIVE_REDIS_TEMPLATE) private Map<Long/*thread id*/,
private ReactiveStringRedisTemplate rt; Map/*LinkedHashMap*/<Long/*time win start*/,
Map<String/*service+apiMethod+apiPath*/, AccessStat>
@Resource >
private GatewayGroupService gatewayGroupService; >
threadTimeWinAccessStatMap = new HashMap<>();
@Override @Override
public Mono<Void> doFilter(ServerWebExchange exchange, Map<String, Object> config, String fixedConfig) { public Mono<Void> doFilter(ServerWebExchange exchange, Map<String, Object> config, String fixedConfig) {
if (statPluginFilterProperties.isStatOpen()) { if (statPluginFilterProperties.isStatOpen()) {
StringBuilder b = ThreadContext.getStringBuilder(); long tid = Thread.currentThread().getId();
b.append(Consts.S.LEFT_BRACE); Map<Long, Map<String, AccessStat>> timeWinAccessStatMap = threadTimeWinAccessStatMap.get(tid);
b.append(ip); toJsonStringValue(b, WebUtils.getOriginIp(exchange)); b.append(Consts.S.COMMA); if (timeWinAccessStatMap == null) {
b.append(gatewayGroup); toJsonStringValue(b, currentGatewayGroups()); b.append(Consts.S.COMMA); timeWinAccessStatMap = new LinkedHashMap<Long, Map<String, AccessStat>>(4, 1) {
b.append(service); toJsonStringValue(b, WebUtils.getClientService(exchange)); b.append(Consts.S.COMMA); @Override
protected boolean removeEldestEntry(Map.Entry eldest) {
return size() > 2;
}
};
threadTimeWinAccessStatMap.put(tid, timeWinAccessStatMap);
}
String appId = WebUtils.getAppId(exchange); long currentTimeWinStart = DateTimeUtils.get10sTimeWinStart(1);
if (appId != null) { Map<String, AccessStat> accessStatMap = timeWinAccessStatMap.computeIfAbsent(currentTimeWinStart, k -> new HashMap<>(128));
b.append(appid); toJsonStringValue(b, appId); b.append(Consts.S.COMMA);
}
b.append(apiMethod); toJsonStringValue(b, exchange.getRequest().getMethodValue()); b.append(Consts.S.COMMA); String service = WebUtils.getClientService(exchange);
b.append(apiPath); toJsonStringValue(b, WebUtils.getClientReqPath(exchange)); b.append(Consts.S.COMMA); String method = exchange.getRequest().getMethodValue();
b.append(reqTime) .append(System.currentTimeMillis()); String path = WebUtils.getClientReqPath(exchange);
b.append(Consts.S.RIGHT_BRACE); String key = ThreadContext.getStringBuilder().append(service).append(method).append(path).toString();
AccessStat accessStat = accessStatMap.get(key);
if (StringUtils.isBlank(statPluginFilterProperties.getFizzAccessStatTopic())) { if (accessStat == null) {
rt.convertAndSend(statPluginFilterProperties.getFizzAccessStatChannel(), b.toString()).subscribe(); accessStat = new AccessStat();
} else { accessStat.service = service;
// log.warn(b.toString(), LogService.HANDLE_STGY, LogService.toKF(statPluginFilterProperties.getFizzAccessStatTopic())); // for internal use accessStat.apiMethod = method;
log.info(b.toString()); accessStat.apiPath = path;
accessStatMap.put(key, accessStat);
}
accessStat.reqTime = System.currentTimeMillis();
accessStat.reqs++;
if (LOGGER.isDebugEnabled()) {
LOGGER.debug("update access stat: {}, which request at {}", accessStat, DateTimeUtils.convert(accessStat.reqTime, Consts.DP.DP19));
} }
} }
return WebUtils.transmitSuccessFilterResultAndEmptyMono(exchange, STAT_PLUGIN_FILTER, null); return WebUtils.transmitSuccessFilterResultAndEmptyMono(exchange, STAT_PLUGIN_FILTER, null);
} }
private String currentGatewayGroups() { public Map<String, AccessStat> getAccessStat(long timeWinStart) {
int sz = gatewayGroupService.currentGatewayGroupSet.size(); Map<String, AccessStat> result = ThreadContext.getHashMap();
if (sz == 1) { threadTimeWinAccessStatMap.forEach(
return gatewayGroupService.currentGatewayGroupSet.iterator().next(); (t, timeWinAccessStatMap) -> {
} Map<String, AccessStat> accessStatMap = timeWinAccessStatMap.get(timeWinStart);
StringBuilder b = ThreadContext.getStringBuilder(ThreadContext.sb0); if (accessStatMap != null) {
byte i = 0; accessStatMap.forEach(
for (String g : gatewayGroupService.currentGatewayGroupSet) { (smp, accessStat) -> {
b.append(g); AccessStat as = result.get(smp);
i++; if (as == null) {
if (i < sz) { accessStat.start = timeWinStart;
b.append(Consts.S.COMMA); result.put(smp, accessStat);
} } else {
} as.reqs = as.reqs + accessStat.reqs;
return b.toString(); if (accessStat.reqTime > as.reqTime) {
} as.reqTime = accessStat.reqTime;
}
private static void toJsonStringValue(StringBuilder b, String value) { }
b.append(Consts.S.DOUBLE_QUOTE).append(value).append(Consts.S.DOUBLE_QUOTE); }
);
}
}
);
return result;
} }
} }

2
fizz-core/src/main/java/we/plugin/stat/StatPluginFilterProperties.java
View File

@@ -37,7 +37,7 @@ public class StatPluginFilterProperties {
@Value("${stat.open:false}") @Value("${stat.open:false}")
private boolean statOpen = false; private boolean statOpen = false;
@Value("${stat.channel:fizz_access_stat}") @Value("${stat.channel:fizz_access_stat_new}")
private String fizzAccessStatChannel; private String fizzAccessStatChannel;
@Value("${stat.topic:}") @Value("${stat.topic:}")