diff --git a/fizz-bootstrap/pom.xml b/fizz-bootstrap/pom.xml index 770a652..59c1ae8 100644 --- a/fizz-bootstrap/pom.xml +++ b/fizz-bootstrap/pom.xml @@ -56,6 +56,13 @@ fizz-spring-boot-starter ${project.version} + + + com.networknt + json-schema-validator-i18n-support + 1.0.39_5 + + + + + repo + file://${project.basedir}/../repo + + + @@ -118,6 +132,12 @@ spring-boot-maven-plugin true + + + com.networknt + json-schema-validator-i18n-support + + diff --git a/fizz-common/src/main/java/we/util/DateTimeUtils.java b/fizz-common/src/main/java/we/util/DateTimeUtils.java index be3a55f..2a9589f 100644 --- a/fizz-common/src/main/java/we/util/DateTimeUtils.java +++ b/fizz-common/src/main/java/we/util/DateTimeUtils.java @@ -17,7 +17,12 @@ package we.util; -import java.time.*; +import we.util.Consts.DP; + +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.ZoneId; import java.time.format.DateTimeFormatter; import java.time.temporal.ChronoField; import java.time.temporal.ChronoUnit; @@ -26,8 +31,6 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; import java.util.stream.Stream; -import we.util.Consts.DP; - /** * @author hongqiaowei */ @@ -165,6 +168,27 @@ public abstract class DateTimeUtils { return localDate1.isEqual(localDate2); } + public static long get10sTimeWinStart(int n) { + LocalDateTime now = LocalDateTime.now().with(ChronoField.MILLI_OF_SECOND, 0); + int sec = now.getSecond(); + long interval; + if (sec > 49) { + interval = sec - 50; + } else if (sec > 39) { + interval = sec - 40; + } else if (sec > 29) { + interval = sec - 30; + } else if (sec > 19) { + interval = sec - 20; + } else if (sec > 9) { + interval = sec - 10; + } else { + interval = sec; + } + long millis = toMillis(now); + return millis - interval * 1000 - (n - 1) * 10L * 1000; + } + /* void iterateBetweenDatesJava8(LocalDate start, LocalDate end) { for (LocalDate date = start; date.isBefore(end); date = date.plusDays(1)) { diff --git a/fizz-core/src/main/java/we/plugin/stat/AccessStat.java b/fizz-core/src/main/java/we/plugin/stat/AccessStat.java new file mode 100644 index 0000000..8f55f60 --- /dev/null +++ b/fizz-core/src/main/java/we/plugin/stat/AccessStat.java @@ -0,0 +1,39 @@ +/* + * Copyright (C) 2021 the original author or authors. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package we.plugin.stat; + +import we.util.JacksonUtils; + +/** + * @author hongqiaowei + */ + +public class AccessStat { + + public String service; + public String apiMethod; + public String apiPath; + public long start; + public int reqs = 0; + public long reqTime; + + @Override + public String toString() { + return JacksonUtils.writeValueAsString(this); + } +} diff --git a/fizz-core/src/main/java/we/plugin/stat/AccessStatSchedConfig.java b/fizz-core/src/main/java/we/plugin/stat/AccessStatSchedConfig.java new file mode 100644 index 0000000..9580357 --- /dev/null +++ b/fizz-core/src/main/java/we/plugin/stat/AccessStatSchedConfig.java @@ -0,0 +1,84 @@ +/* + * Copyright (C) 2021 the original author or authors. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package we.plugin.stat; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.context.annotation.Configuration; +import org.springframework.data.redis.core.ReactiveStringRedisTemplate; +import org.springframework.scheduling.annotation.Scheduled; +import we.config.AggregateRedisConfig; +import we.config.SchedConfig; +import we.util.Consts; +import we.util.DateTimeUtils; +import we.util.StringUtils; + +import javax.annotation.Resource; +import java.util.Map; + +/** + * @author hongqiaowei + */ + +@Configuration +public class AccessStatSchedConfig extends SchedConfig { + + private static final Logger LOGGER = LoggerFactory.getLogger(AccessStatSchedConfig.class); + + private static final Logger STAT_LOGGER = LoggerFactory.getLogger("stat"); + + @Resource(name = AggregateRedisConfig.AGGREGATE_REACTIVE_REDIS_TEMPLATE) + private ReactiveStringRedisTemplate rt; + + @Resource + private StatPluginFilterProperties statPluginFilterProperties; + + @Resource + private StatPluginFilter statPluginFilter; + + @Scheduled(cron = "${fizz-access-stat-sched.cron:2/10 * * * * ?}") + public void sched() { + long prevTimeWinStart = DateTimeUtils.get10sTimeWinStart(2); + Map accessStatMap = statPluginFilter.getAccessStat(prevTimeWinStart); + + if (accessStatMap.isEmpty()) { + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("no access stat in {} window", DateTimeUtils.convert(prevTimeWinStart, Consts.DP.DP19)); + } + } else { + accessStatMap.forEach( + (smp, accessStat) -> { + String msg = accessStat.toString(); + String topic = statPluginFilterProperties.getFizzAccessStatTopic(); + if (StringUtils.isBlank(topic)) { + String channel = statPluginFilterProperties.getFizzAccessStatChannel(); + rt.convertAndSend(channel, msg).subscribe(); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("send access stat {} which belong to {} window to channel {}", msg, DateTimeUtils.convert(accessStat.start, Consts.DP.DP19), channel); + } + } else { + STAT_LOGGER.info(msg); + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("send access stat {} which belong to {} window to topic", msg, DateTimeUtils.convert(accessStat.start, Consts.DP.DP19)); + } + } + } + ); + } + } +} diff --git a/fizz-core/src/main/java/we/plugin/stat/StatPluginFilter.java b/fizz-core/src/main/java/we/plugin/stat/StatPluginFilter.java index b021700..f6cbd4e 100644 --- a/fizz-core/src/main/java/we/plugin/stat/StatPluginFilter.java +++ b/fizz-core/src/main/java/we/plugin/stat/StatPluginFilter.java @@ -17,21 +17,20 @@ package we.plugin.stat; -import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.data.redis.core.ReactiveStringRedisTemplate; import org.springframework.stereotype.Component; import org.springframework.web.server.ServerWebExchange; import reactor.core.publisher.Mono; -import we.config.AggregateRedisConfig; import we.plugin.PluginFilter; -import we.plugin.auth.GatewayGroupService; import we.util.Consts; +import we.util.DateTimeUtils; import we.util.ThreadContext; import we.util.WebUtils; import javax.annotation.Resource; +import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.Map; /** @@ -42,82 +41,84 @@ import java.util.Map; @Component(StatPluginFilter.STAT_PLUGIN_FILTER) public class StatPluginFilter extends PluginFilter { - private static final Logger log = LoggerFactory.getLogger("stat"); + private static final Logger LOGGER = LoggerFactory.getLogger(StatPluginFilter.class); public static final String STAT_PLUGIN_FILTER = "statPlugin"; - private static final String ip = "\"ip\":"; - - private static final String gatewayGroup = "\"gatewayGroup\":"; - - private static final String service = "\"service\":"; - - private static final String appid = "\"appid\":"; - - private static final String apiMethod = "\"apiMethod\":"; - - private static final String apiPath = "\"apiPath\":"; - - private static final String reqTime = "\"reqTime\":"; - @Resource private StatPluginFilterProperties statPluginFilterProperties; - @Resource(name = AggregateRedisConfig.AGGREGATE_REACTIVE_REDIS_TEMPLATE) - private ReactiveStringRedisTemplate rt; - - @Resource - private GatewayGroupService gatewayGroupService; + private Map + > + > + threadTimeWinAccessStatMap = new HashMap<>(); @Override public Mono doFilter(ServerWebExchange exchange, Map config, String fixedConfig) { if (statPluginFilterProperties.isStatOpen()) { - StringBuilder b = ThreadContext.getStringBuilder(); - b.append(Consts.S.LEFT_BRACE); - b.append(ip); toJsonStringValue(b, WebUtils.getOriginIp(exchange)); b.append(Consts.S.COMMA); - b.append(gatewayGroup); toJsonStringValue(b, currentGatewayGroups()); b.append(Consts.S.COMMA); - b.append(service); toJsonStringValue(b, WebUtils.getClientService(exchange)); b.append(Consts.S.COMMA); + long tid = Thread.currentThread().getId(); + Map> timeWinAccessStatMap = threadTimeWinAccessStatMap.get(tid); + if (timeWinAccessStatMap == null) { + timeWinAccessStatMap = new LinkedHashMap>(4, 1) { + @Override + protected boolean removeEldestEntry(Map.Entry eldest) { + return size() > 2; + } + }; + threadTimeWinAccessStatMap.put(tid, timeWinAccessStatMap); + } - String appId = WebUtils.getAppId(exchange); - if (appId != null) { - b.append(appid); toJsonStringValue(b, appId); b.append(Consts.S.COMMA); - } + long currentTimeWinStart = DateTimeUtils.get10sTimeWinStart(1); + Map accessStatMap = timeWinAccessStatMap.computeIfAbsent(currentTimeWinStart, k -> new HashMap<>(128)); - b.append(apiMethod); toJsonStringValue(b, exchange.getRequest().getMethodValue()); b.append(Consts.S.COMMA); - b.append(apiPath); toJsonStringValue(b, WebUtils.getClientReqPath(exchange)); b.append(Consts.S.COMMA); - b.append(reqTime) .append(System.currentTimeMillis()); - b.append(Consts.S.RIGHT_BRACE); - - if (StringUtils.isBlank(statPluginFilterProperties.getFizzAccessStatTopic())) { - rt.convertAndSend(statPluginFilterProperties.getFizzAccessStatChannel(), b.toString()).subscribe(); - } else { - // log.warn(b.toString(), LogService.HANDLE_STGY, LogService.toKF(statPluginFilterProperties.getFizzAccessStatTopic())); // for internal use - log.info(b.toString()); + String service = WebUtils.getClientService(exchange); + String method = exchange.getRequest().getMethodValue(); + String path = WebUtils.getClientReqPath(exchange); + String key = ThreadContext.getStringBuilder().append(service).append(method).append(path).toString(); + AccessStat accessStat = accessStatMap.get(key); + if (accessStat == null) { + accessStat = new AccessStat(); + accessStat.service = service; + accessStat.apiMethod = method; + accessStat.apiPath = path; + accessStatMap.put(key, accessStat); + } + accessStat.reqTime = System.currentTimeMillis(); + accessStat.reqs++; + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("update access stat: {}, which request at {}", accessStat, DateTimeUtils.convert(accessStat.reqTime, Consts.DP.DP19)); } } return WebUtils.transmitSuccessFilterResultAndEmptyMono(exchange, STAT_PLUGIN_FILTER, null); } - private String currentGatewayGroups() { - int sz = gatewayGroupService.currentGatewayGroupSet.size(); - if (sz == 1) { - return gatewayGroupService.currentGatewayGroupSet.iterator().next(); - } - StringBuilder b = ThreadContext.getStringBuilder(ThreadContext.sb0); - byte i = 0; - for (String g : gatewayGroupService.currentGatewayGroupSet) { - b.append(g); - i++; - if (i < sz) { - b.append(Consts.S.COMMA); - } - } - return b.toString(); - } - - private static void toJsonStringValue(StringBuilder b, String value) { - b.append(Consts.S.DOUBLE_QUOTE).append(value).append(Consts.S.DOUBLE_QUOTE); + public Map getAccessStat(long timeWinStart) { + Map result = ThreadContext.getHashMap(); + threadTimeWinAccessStatMap.forEach( + (t, timeWinAccessStatMap) -> { + Map accessStatMap = timeWinAccessStatMap.get(timeWinStart); + if (accessStatMap != null) { + accessStatMap.forEach( + (smp, accessStat) -> { + AccessStat as = result.get(smp); + if (as == null) { + accessStat.start = timeWinStart; + result.put(smp, accessStat); + } else { + as.reqs = as.reqs + accessStat.reqs; + if (accessStat.reqTime > as.reqTime) { + as.reqTime = accessStat.reqTime; + } + } + } + ); + } + } + ); + return result; } } diff --git a/fizz-core/src/main/java/we/plugin/stat/StatPluginFilterProperties.java b/fizz-core/src/main/java/we/plugin/stat/StatPluginFilterProperties.java index cd8b471..3697196 100644 --- a/fizz-core/src/main/java/we/plugin/stat/StatPluginFilterProperties.java +++ b/fizz-core/src/main/java/we/plugin/stat/StatPluginFilterProperties.java @@ -37,7 +37,7 @@ public class StatPluginFilterProperties { @Value("${stat.open:false}") private boolean statOpen = false; - @Value("${stat.channel:fizz_access_stat}") + @Value("${stat.channel:fizz_access_stat_new}") private String fizzAccessStatChannel; @Value("${stat.topic:}")