From 5df2cf383430f631a102b6eda5e3f599527c2cbc Mon Sep 17 00:00:00 2001 From: hongqiaowei Date: Fri, 4 Mar 2022 11:18:44 +0800 Subject: [PATCH] Init 2.6 --- fizz-bootstrap/pom.xml | 6 +- fizz-common/pom.xml | 11 +- .../src/main/java/we/config/SchedConfig.java | 4 +- .../src/main/java/we/util/StringUtils.java | 34 +++ fizz-core/pom.xml | 2 +- .../config/FizzBeanFactoryPostProcessor.java | 281 ++++++++++++++++++ .../factory/config/FizzBeanPostProcessor.java | 47 +++ .../AggregateRedisConfigProperties.java | 3 +- .../we/config/FizzConfigConfiguration.java | 104 +++++++ .../FizzGatewayNodeStatSchedConfig.java | 91 ++++++ .../java/we/config/FlowStatSchedConfig.java | 26 +- .../src/main/java/we/config/SystemConfig.java | 35 +-- .../config/annotation/FizzRefreshScope.java | 37 +++ .../event/FizzApplicationListener.java | 46 +++ .../we/context/event/FizzRefreshEvent.java | 53 ++++ .../event/FizzRefreshEventListener.java | 104 +++++++ .../scope/refresh/FizzRefreshScope.java | 121 ++++++++ .../FizzRefreshScopeRefreshedEvent.java | 44 +++ .../controller/ManagerConfigController.java | 215 ++++++++++++++ .../controller/req/GetApiConfigDetailReq.java | 41 +++ .../we/controller/req/GetApiConfigReq.java | 72 +++++ .../we/controller/resp/ApiConfigInfo.java | 69 +++++ .../resp/GetApiConfigDetailResp.java | 155 ++++++++++ .../we/controller/resp/GetApiConfigResp.java | 43 +++ .../filter/FilterExceptionHandlerConfig.java | 2 + .../java/we/filter/FlowControlFilter.java | 54 +++- .../java/we/monitor/FizzMonitorService.java | 91 ++++++ .../java/we/plugin/auth/ApiConfigService.java | 4 + .../stat/StatPluginFilterProperties.java | 3 +- .../ApacheDubboGenericServiceProperties.java | 3 +- .../src/main/java/we/stats/FlowStat.java | 20 +- .../src/main/java/we/stats/ResourceStat.java | 33 +- .../src/main/java/we/stats/TimeSlot.java | 24 ++ .../main/java/we/stats/TimeWindowStat.java | 36 +++ .../ratelimit/ResourceRateLimitConfig.java | 1 + fizz-core/src/main/java/we/util/WebUtils.java | 2 + .../src/test/java/we/stats/FlowStatTests.java | 5 +- fizz-plugin/pom.xml | 2 +- fizz-spring-boot-starter/pom.xml | 2 +- .../main/resources/META-INF/spring.factories | 3 + pom.xml | 8 +- 41 files changed, 1886 insertions(+), 51 deletions(-) create mode 100644 fizz-common/src/main/java/we/util/StringUtils.java create mode 100644 fizz-core/src/main/java/we/beans/factory/config/FizzBeanFactoryPostProcessor.java create mode 100644 fizz-core/src/main/java/we/beans/factory/config/FizzBeanPostProcessor.java create mode 100644 fizz-core/src/main/java/we/config/FizzConfigConfiguration.java create mode 100644 fizz-core/src/main/java/we/config/FizzGatewayNodeStatSchedConfig.java create mode 100644 fizz-core/src/main/java/we/context/config/annotation/FizzRefreshScope.java create mode 100644 fizz-core/src/main/java/we/context/event/FizzApplicationListener.java create mode 100644 fizz-core/src/main/java/we/context/event/FizzRefreshEvent.java create mode 100644 fizz-core/src/main/java/we/context/event/FizzRefreshEventListener.java create mode 100644 fizz-core/src/main/java/we/context/scope/refresh/FizzRefreshScope.java create mode 100644 fizz-core/src/main/java/we/context/scope/refresh/FizzRefreshScopeRefreshedEvent.java create mode 100644 fizz-core/src/main/java/we/controller/req/GetApiConfigDetailReq.java create mode 100644 fizz-core/src/main/java/we/controller/req/GetApiConfigReq.java create mode 100644 fizz-core/src/main/java/we/controller/resp/ApiConfigInfo.java create mode 100644 fizz-core/src/main/java/we/controller/resp/GetApiConfigDetailResp.java create mode 100644 fizz-core/src/main/java/we/controller/resp/GetApiConfigResp.java create mode 100644 fizz-core/src/main/java/we/monitor/FizzMonitorService.java diff --git a/fizz-bootstrap/pom.xml b/fizz-bootstrap/pom.xml index a33f800..d4f0d6b 100644 --- a/fizz-bootstrap/pom.xml +++ b/fizz-bootstrap/pom.xml @@ -12,7 +12,7 @@ com.fizzgate fizz-bootstrap - 2.5.2 + 2.6 1.8 @@ -22,7 +22,7 @@ 5.3.7.RELEASE 4.1.74.Final 4.4.15 - 2.17.1 + 2.17.2 1.7.36 3.12.0 1.18.22 @@ -34,7 +34,7 @@ 1.15 2.11.1 2.8.9 - 2.0.48.Final + 2.0.50.Final 2.2.9.RELEASE 1.7.1 1.30 diff --git a/fizz-common/pom.xml b/fizz-common/pom.xml index 7ef725c..6215c3b 100644 --- a/fizz-common/pom.xml +++ b/fizz-common/pom.xml @@ -5,7 +5,7 @@ fizz-gateway-community com.fizzgate - 2.5.2 + 2.6 ../pom.xml 4.0.0 @@ -17,6 +17,15 @@ + + org.springframework.cloud + spring-cloud-context + + + org.springframework.cloud + spring-cloud-commons + + com.networknt json-schema-validator-i18n-support diff --git a/fizz-common/src/main/java/we/config/SchedConfig.java b/fizz-common/src/main/java/we/config/SchedConfig.java index 80e0e8d..8616d93 100644 --- a/fizz-common/src/main/java/we/config/SchedConfig.java +++ b/fizz-common/src/main/java/we/config/SchedConfig.java @@ -25,6 +25,7 @@ import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.context.annotation.Bean; import org.springframework.scheduling.Trigger; import org.springframework.scheduling.TriggerContext; +import org.springframework.scheduling.annotation.EnableScheduling; import org.springframework.scheduling.annotation.SchedulingConfigurer; import org.springframework.scheduling.config.ScheduledTaskRegistrar; @@ -33,9 +34,10 @@ import org.springframework.scheduling.config.ScheduledTaskRegistrar; */ @ConfigurationProperties(prefix = "sched") +@EnableScheduling public abstract class SchedConfig implements SchedulingConfigurer { - private int executors = 1; + private int executors = 2; public void setExecutors(int es) { executors = es; diff --git a/fizz-common/src/main/java/we/util/StringUtils.java b/fizz-common/src/main/java/we/util/StringUtils.java new file mode 100644 index 0000000..776ad56 --- /dev/null +++ b/fizz-common/src/main/java/we/util/StringUtils.java @@ -0,0 +1,34 @@ +/* + * Copyright (C) 2021 the original author or authors. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package we.util; + +import java.util.Objects; + +/** + * @author hongqiaowei + */ + +public abstract class StringUtils extends org.apache.commons.lang3.StringUtils { + + private StringUtils() { + } + + public static boolean notEquals(String s1, String s2) { + return !Objects.equals(s1, s2); + } +} diff --git a/fizz-core/pom.xml b/fizz-core/pom.xml index b79cf0c..4c1e280 100644 --- a/fizz-core/pom.xml +++ b/fizz-core/pom.xml @@ -5,7 +5,7 @@ fizz-gateway-community com.fizzgate - 2.5.2 + 2.6 ../pom.xml 4.0.0 diff --git a/fizz-core/src/main/java/we/beans/factory/config/FizzBeanFactoryPostProcessor.java b/fizz-core/src/main/java/we/beans/factory/config/FizzBeanFactoryPostProcessor.java new file mode 100644 index 0000000..4f1c269 --- /dev/null +++ b/fizz-core/src/main/java/we/beans/factory/config/FizzBeanFactoryPostProcessor.java @@ -0,0 +1,281 @@ +/* + * Copyright (C) 2021 the original author or authors. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package we.beans.factory.config; + +import com.fasterxml.jackson.core.type.TypeReference; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.BeansException; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.beans.factory.config.BeanFactoryPostProcessor; +import org.springframework.beans.factory.config.ConfigurableListableBeanFactory; +import org.springframework.beans.factory.support.AbstractBeanDefinition; +import org.springframework.cloud.context.scope.GenericScope; +import org.springframework.context.ApplicationContext; +import org.springframework.context.ApplicationContextAware; +import org.springframework.context.EnvironmentAware; +import org.springframework.core.Ordered; +import org.springframework.core.env.ConfigurableEnvironment; +import org.springframework.core.env.Environment; +import org.springframework.core.env.MapPropertySource; +import org.springframework.core.env.MutablePropertySources; +import org.springframework.data.redis.core.ReactiveStringRedisTemplate; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; +import we.config.FizzConfigConfiguration; +import we.context.config.annotation.FizzRefreshScope; +import we.context.event.FizzRefreshEvent; +import we.global_resource.GlobalResource; +import we.util.*; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; + +/** + * @author hongqiaowei + */ + +public class FizzBeanFactoryPostProcessor implements BeanFactoryPostProcessor, EnvironmentAware, ApplicationContextAware, Ordered { + + private static final Logger LOGGER = LoggerFactory.getLogger(FizzBeanFactoryPostProcessor.class); + + private ApplicationContext applicationContext; + + private ConfigurableEnvironment environment; + + private final Map property2beanMap = new HashMap<>(32); + + private ReactiveStringRedisTemplate reactiveStringRedisTemplate; + + @Override + public void postProcessBeanFactory(ConfigurableListableBeanFactory beanFactory) throws BeansException { + String fizzConfigEnable = environment.getProperty("fizz.config.enable", Consts.S.TRUE); + if (fizzConfigEnable.equals(Consts.S.TRUE)) { + initReactiveStringRedisTemplate(); + initFizzPropertySource(); + initBeanProperty2beanMap(beanFactory); + } + } + + private void initReactiveStringRedisTemplate() { + String host = environment.getProperty("aggregate.redis.host"); + String port = environment.getProperty("aggregate.redis.port"); + String password = environment.getProperty("aggregate.redis.password"); + String database = environment.getProperty("aggregate.redis.database"); + reactiveStringRedisTemplate = ReactiveRedisHelper.getStringRedisTemplate(host, Integer.parseInt(port), password, Integer.parseInt(database)); + } + + private void initFizzPropertySource() { + MutablePropertySources propertySources = environment.getPropertySources(); + Map sources = new HashMap<>(); + MapPropertySource fizzPropertySource = new MapPropertySource(FizzConfigConfiguration.PROPERTY_SOURCE, sources); + propertySources.addFirst(fizzPropertySource); + + Result result = Result.succ(); + Flux> fizzConfigs = reactiveStringRedisTemplate.opsForHash().entries("fizz_config"); + fizzConfigs.collectList() + .defaultIfEmpty(Collections.emptyList()) + .flatMap( + es -> { + if (es.isEmpty()) { + LOGGER.info("no fizz configs"); + } else { + String value = null; + try { + for (Map.Entry e : es) { + String key = (String) e.getKey(); + value = (String) e.getValue(); + Map config = JacksonUtils.readValue(value, new TypeReference>(){}); + sources.put(key, config.get(key)); + } + } catch (Throwable t) { + result.code = Result.FAIL; + result.msg = "init fizz configs error, json: " + value; + result.t = t; + } + } + return Mono.empty(); + } + ) + .onErrorReturn( + throwable -> { + result.code = Result.FAIL; + result.msg = "init fizz configs error"; + result.t = throwable; + return true; + }, + result + ) + .block(); + + if (result.code == Result.FAIL) { + throw new RuntimeException(result.msg, result.t); + } + if (!sources.isEmpty()) { + LOGGER.info("fizz configs: {}", JacksonUtils.writeValueAsString(sources)); + } + + String channel = "fizz_config_channel"; + reactiveStringRedisTemplate.listenToChannel(channel) + .doOnError( + t -> { + result.code = Result.FAIL; + result.msg = "lsn " + channel + " channel error"; + result.t = t; + LOGGER.error("lsn channel {} error", channel, t); + } + ) + .doOnSubscribe( + s -> { + LOGGER.info("success to lsn on {}", channel); + } + ) + .doOnNext( + msg -> { + String message = msg.getMessage(); + try { + /*Map changes = JacksonUtils.readValue(message, new TypeReference>(){}); + boolean defaultConfigEnable = false; + String defaultConfigEnableStr = changes.get("fizz.default-config.enable"); + if (defaultConfigEnableStr == null) { + defaultConfigEnable = environment.getProperty("fizz.default-config.enable", Boolean.class, false); + } else { + defaultConfigEnable = Boolean.parseBoolean(defaultConfigEnableStr); + } + boolean finalDefaultConfigEnable = defaultConfigEnable; + changes.forEach( + (property, value) -> { + if (StringUtils.isBlank(value)) { + if (finalDefaultConfigEnable) { + Object v = FizzConfigConfiguration.DEFAULT_CONFIG_MAP.get(property); + if (v == null) { + sources.remove(property); + } else { + sources.put(property, v); + } + } else { + sources.remove(property); + } + } else { + sources.put(property, value); + } + } + );*/ + Map change = JacksonUtils.readValue(message, new TypeReference>(){}); + int isDeleted = (int) change.remove("isDeleted"); + Map.Entry propertyValue = change.entrySet().iterator().next(); + String property = propertyValue.getKey(); + if (isDeleted == 1) { + sources.remove(property); + } else { + sources.put(property, propertyValue.getValue()); + } + LOGGER.info("new fizz configs: {}", JacksonUtils.writeValueAsString(sources)); + FizzRefreshEvent refreshEvent = new FizzRefreshEvent(applicationContext, FizzRefreshEvent.ENV_CHANGE, change); + applicationContext.publishEvent(refreshEvent); + } catch (Throwable t) { + LOGGER.error("update fizz config {} error", message, t); + } + } + ) + .subscribe(); + + if (result.code == Result.FAIL) { + throw new RuntimeException(result.msg, result.t); + } + } + + private void initBeanProperty2beanMap(ConfigurableListableBeanFactory beanFactory) { + ClassLoader beanClassLoader = beanFactory.getBeanClassLoader(); + Iterator beanNamesIterator = beanFactory.getBeanNamesIterator(); + while (beanNamesIterator.hasNext()) { + String beanName = beanNamesIterator.next(); + if (beanName.startsWith(GenericScope.SCOPED_TARGET_PREFIX)) { + AbstractBeanDefinition beanDefinition = (AbstractBeanDefinition) beanFactory.getBeanDefinition(beanName); + try { + beanDefinition.resolveBeanClass(beanClassLoader); + } catch (ClassNotFoundException e) { + throw new RuntimeException(e); + } + + Class beanClass = null; + try { + beanClass = beanDefinition.getBeanClass(); + } catch (IllegalStateException e) { + LOGGER.warn("get {} bean class exception: {}", beanName, e.getMessage()); + continue; + } + + FizzRefreshScope an = beanClass.getAnnotation(FizzRefreshScope.class); + if (an != null) { + ReflectionUtils.doWithFields( + beanClass, + field -> { + Value annotation = field.getAnnotation(Value.class); + if (annotation != null) { + property2beanMap.put(extractPlaceholderKey(annotation.value()), beanName); + } + } + ); + ReflectionUtils.doWithMethods( + beanClass, + method -> { + Value annotation = method.getAnnotation(Value.class); + if (annotation != null) { + property2beanMap.put(extractPlaceholderKey(annotation.value()), beanName); + } + } + ); + } + } + } + + LOGGER.info("fizz refresh scope property to bean map: {}", JacksonUtils.writeValueAsString(property2beanMap)); + } + + private String extractPlaceholderKey(String propertyPlaceholder) { + int begin = 2; + int end = propertyPlaceholder.indexOf(':'); + if (end < 0) { + end = propertyPlaceholder.indexOf('}'); + } + return propertyPlaceholder.substring(begin, end); + } + + @Override + public void setApplicationContext(ApplicationContext applicationContext) throws BeansException { + this.applicationContext = applicationContext; + } + + @Override + public void setEnvironment(Environment environment) { + this.environment = (ConfigurableEnvironment) environment; + } + + @Override + public int getOrder() { + return Ordered.LOWEST_PRECEDENCE; + } + + public String getBean(String property) { + return property2beanMap.get(property); + } +} diff --git a/fizz-core/src/main/java/we/beans/factory/config/FizzBeanPostProcessor.java b/fizz-core/src/main/java/we/beans/factory/config/FizzBeanPostProcessor.java new file mode 100644 index 0000000..52b3b3f --- /dev/null +++ b/fizz-core/src/main/java/we/beans/factory/config/FizzBeanPostProcessor.java @@ -0,0 +1,47 @@ +/* + * Copyright (C) 2021 the original author or authors. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package we.beans.factory.config; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.BeansException; +import org.springframework.beans.factory.config.BeanPostProcessor; +import org.springframework.core.Ordered; + +/** + * @author hongqiaowei + */ + +public class FizzBeanPostProcessor implements BeanPostProcessor, Ordered { + + private static final Logger LOGGER = LoggerFactory.getLogger(FizzBeanPostProcessor.class); + + @Override + public int getOrder() { + return Ordered.LOWEST_PRECEDENCE; + } + + @Override + public Object postProcessBeforeInitialization(Object bean, String beanName) throws BeansException { + return bean; + } + + public Object postProcessAfterInitialization(Object bean, String beanName) throws BeansException { + return bean; + } +} diff --git a/fizz-core/src/main/java/we/config/AggregateRedisConfigProperties.java b/fizz-core/src/main/java/we/config/AggregateRedisConfigProperties.java index f94e51c..6d50624 100644 --- a/fizz-core/src/main/java/we/config/AggregateRedisConfigProperties.java +++ b/fizz-core/src/main/java/we/config/AggregateRedisConfigProperties.java @@ -21,6 +21,7 @@ import lombok.Data; import org.springframework.beans.factory.annotation.Value; import org.springframework.cloud.context.config.annotation.RefreshScope; import org.springframework.stereotype.Component; +import we.context.config.annotation.FizzRefreshScope; /** * {@link AggregateRedisConfig} properties @@ -28,7 +29,7 @@ import org.springframework.stereotype.Component; * @author zhongjie */ -@RefreshScope +@FizzRefreshScope @Component @Data public class AggregateRedisConfigProperties { diff --git a/fizz-core/src/main/java/we/config/FizzConfigConfiguration.java b/fizz-core/src/main/java/we/config/FizzConfigConfiguration.java new file mode 100644 index 0000000..631b9d9 --- /dev/null +++ b/fizz-core/src/main/java/we/config/FizzConfigConfiguration.java @@ -0,0 +1,104 @@ +/* + * Copyright (C) 2021 the original author or authors. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package we.config; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import we.beans.factory.config.FizzBeanFactoryPostProcessor; +import we.context.event.FizzRefreshEventListener; +import we.context.scope.refresh.FizzRefreshScope; + +import java.util.HashMap; +import java.util.Map; + +/** + * @author hongqiaowei + */ + +@Configuration(proxyBeanMethods = false) +public class FizzConfigConfiguration { + + public static final String PROPERTY_SOURCE = "FizzPropertySource"; + + public static final String REFRESH_SCOPE_NAME = "FizzRefresh"; + + /*public static final Map DEFAULT_CONFIG_MAP = new HashMap<>(); + + static { + DEFAULT_CONFIG_MAP.put("server.port", 8600); + DEFAULT_CONFIG_MAP.put("gateway.prefix", "/proxy"); + DEFAULT_CONFIG_MAP.put("server.fileUpload.maxDiskUsagePerPart", 104857600); + DEFAULT_CONFIG_MAP.put("cors", true); + + DEFAULT_CONFIG_MAP.put("fizz.error.response.http-status.enable", true); + DEFAULT_CONFIG_MAP.put("fizz.error.response.code-field", "msgCode"); + DEFAULT_CONFIG_MAP.put("fizz.error.response.message-field", "message"); + + DEFAULT_CONFIG_MAP.put("fizz-trace-id.header", "X-Trace-Id"); + DEFAULT_CONFIG_MAP.put("fizz-trace-id.value-strategy", "requestId"); + DEFAULT_CONFIG_MAP.put("fizz-trace-id.value-prefix", "fizz"); + + DEFAULT_CONFIG_MAP.put("custom.header.appid", "fizz-appid"); + DEFAULT_CONFIG_MAP.put("custom.header.sign", "fizz-sign"); + DEFAULT_CONFIG_MAP.put("custom.header.ts", "fizz-ts"); + + DEFAULT_CONFIG_MAP.put("proxy-webclient.chTcpNodelay", true); + DEFAULT_CONFIG_MAP.put("proxy-webclient.chSoKeepAlive", true); + DEFAULT_CONFIG_MAP.put("proxy-webclient.chConnTimeout", 60000); + DEFAULT_CONFIG_MAP.put("proxy-webclient.connReadTimeout", 60000); + DEFAULT_CONFIG_MAP.put("proxy-webclient.connWriteTimeout", 60000); + DEFAULT_CONFIG_MAP.put("proxy-webclient.compress", true); + DEFAULT_CONFIG_MAP.put("proxy-webclient.trustInsecureSSL", true); + + DEFAULT_CONFIG_MAP.put("stat.open", true); + DEFAULT_CONFIG_MAP.put("send-log.open", true); + DEFAULT_CONFIG_MAP.put("log.headers", "COOKIE,FIZZ-APPID,FIZZ-SIGN,FIZZ-TS,FIZZ-RSV,HOST"); + + DEFAULT_CONFIG_MAP.put("fizz.aggregate.writeMapNullValue", false); + DEFAULT_CONFIG_MAP.put("gateway.aggr.proxy_set_headers", "X-Real-IP,X-Forwarded-Proto,X-Forwarded-For"); + DEFAULT_CONFIG_MAP.put("fizz-dubbo-client.address", "zookeeper://127.0.0.1:2181"); + + DEFAULT_CONFIG_MAP.put("fizz.dedicated-line.server.enable", true); + DEFAULT_CONFIG_MAP.put("fizz.dedicated-line.client.enable", true); + DEFAULT_CONFIG_MAP.put("fizz.dedicated-line.client.port", 8601); + DEFAULT_CONFIG_MAP.put("fizz.dedicated-line.client.request.timeliness", 300); + DEFAULT_CONFIG_MAP.put("fizz.dedicated-line.client.request.timeout", 0); + DEFAULT_CONFIG_MAP.put("fizz.dedicated-line.client.request.retry-count", 0); + DEFAULT_CONFIG_MAP.put("fizz.dedicated-line.client.request.retry-interval", 0); + }*/ + + @Bean + public FizzBeanFactoryPostProcessor fizzBeanFactoryPostProcessor() { + return new FizzBeanFactoryPostProcessor(); + } + + /*@Bean + public FizzBeanPostProcessor fizzBeanPostProcessor() { + return new FizzBeanPostProcessor(); + }*/ + + @Bean + public static FizzRefreshScope fizzRefreshScope() { + return new FizzRefreshScope(); + } + + @Bean + public FizzRefreshEventListener fizzRefreshEventListener(FizzBeanFactoryPostProcessor fizzBeanFactoryPostProcessor, FizzRefreshScope scope) { + return new FizzRefreshEventListener(fizzBeanFactoryPostProcessor, scope); + } +} diff --git a/fizz-core/src/main/java/we/config/FizzGatewayNodeStatSchedConfig.java b/fizz-core/src/main/java/we/config/FizzGatewayNodeStatSchedConfig.java new file mode 100644 index 0000000..8d28fa9 --- /dev/null +++ b/fizz-core/src/main/java/we/config/FizzGatewayNodeStatSchedConfig.java @@ -0,0 +1,91 @@ +/* + * Copyright (C) 2021 the original author or authors. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package we.config; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.boot.web.reactive.context.ReactiveWebServerApplicationContext; +import org.springframework.context.annotation.Configuration; +import org.springframework.data.redis.core.ReactiveStringRedisTemplate; +import org.springframework.scheduling.annotation.EnableScheduling; +import org.springframework.scheduling.annotation.Scheduled; +import we.util.JacksonUtils; +import we.util.NetworkUtils; + +import javax.annotation.PostConstruct; +import javax.annotation.Resource; +import java.lang.management.ManagementFactory; + +/** + * @author hongqiaowei + */ + +@Configuration +//@EnableScheduling +public class FizzGatewayNodeStatSchedConfig extends SchedConfig { + + private final static class Stat { + public String serviceName; + public String ip; + public int port; + public long ts; + public long startTs; + } + + private static final Logger LOGGER = LoggerFactory.getLogger(FizzGatewayNodeStatSchedConfig.class); + + private static final String fizz_gateway_nodes = "fizz_gateway_nodes"; + + @Resource + private ReactiveWebServerApplicationContext applicationContext; + + @Resource(name = AggregateRedisConfig.AGGREGATE_REACTIVE_REDIS_TEMPLATE) + private ReactiveStringRedisTemplate rt; + + private Stat stat = new Stat(); + + private String hashKey; + + @PostConstruct + public void postConstruct() { + stat.serviceName = applicationContext.getApplicationName(); + stat.ip = NetworkUtils.getServerIp(); + stat.port = Integer.parseInt(applicationContext.getEnvironment().getProperty("server.port", "8600")); + hashKey = stat.ip + ':' + stat.port; + stat.startTs = ManagementFactory.getRuntimeMXBean().getStartTime(); + } + + @Scheduled(cron = "${fizz-gateway-node-stat-sched.cron:*/3 * * * * ?}") + public void sched() { + stat.ts = System.currentTimeMillis(); + String s; + try { + s = JacksonUtils.writeValueAsString(stat); + } catch (RuntimeException e) { + LOGGER.error("serial fizz gateway node stat error", e); + return; + } + rt.opsForHash().put(fizz_gateway_nodes, hashKey, s) + .doOnError( + t -> { + LOGGER.error("report fizz gateway node stat error", t); + } + ) + .block(); + } +} diff --git a/fizz-core/src/main/java/we/config/FlowStatSchedConfig.java b/fizz-core/src/main/java/we/config/FlowStatSchedConfig.java index 8bc2025..80c9df4 100644 --- a/fizz-core/src/main/java/we/config/FlowStatSchedConfig.java +++ b/fizz-core/src/main/java/we/config/FlowStatSchedConfig.java @@ -45,7 +45,7 @@ import java.util.List; */ @Configuration -@EnableScheduling +//@EnableScheduling public class FlowStatSchedConfig extends SchedConfig { private static final Logger log = LoggerFactory.getLogger(FlowStatSchedConfig.class); @@ -72,6 +72,11 @@ public class FlowStatSchedConfig extends SchedConfig { private static final String _path = "\"path\":"; private static final String _peakRps = "\"peakRps\":"; + private static final String _2xxStatus = "\"status2xxs\":"; + private static final String _4xxStatus = "\"status4xxs\":"; + private static final String _5xxStatus = "\"status5xxs\":"; + private static final String _504Status = "\"status504s\":"; + private static final String parentResourceList = "$prl"; @Resource @@ -118,9 +123,12 @@ public class FlowStatSchedConfig extends SchedConfig { int type = ResourceRateLimitConfig.Type.NODE, id = 0; ResourceRateLimitConfig c = resourceRateLimitConfigService.getResourceRateLimitConfig(resource); - if (c == null) { // _global, service, app, app+service, ip, ip+service + if (c == null) { // _global, host, service, app, app+service, ip, ip+service node = ResourceIdUtils.getNode(resource); - if (node != null && node.equals(ResourceIdUtils.NODE)) { + if (node != null) { + if (!node.equals(ResourceIdUtils.NODE)) { + type = ResourceRateLimitConfig.Type.HOST; + } } else { service = ResourceIdUtils.getService(resource); app = ResourceIdUtils.getApp(resource); @@ -183,8 +191,8 @@ public class FlowStatSchedConfig extends SchedConfig { b.append(_id); b.append(id); b.append(Consts.S.COMMA); String r = null; - if (type == ResourceRateLimitConfig.Type.NODE) { - r = ResourceIdUtils.NODE; + if (type == ResourceRateLimitConfig.Type.NODE || type == ResourceRateLimitConfig.Type.HOST) { + r = node; } else if (type == ResourceRateLimitConfig.Type.SERVICE_DEFAULT || type == ResourceRateLimitConfig.Type.SERVICE) { r = service; } @@ -221,7 +229,13 @@ public class FlowStatSchedConfig extends SchedConfig { b.append(_errors); b.append(w.getErrors()); b.append(Consts.S.COMMA); b.append(_avgRespTime); b.append(w.getAvgRt()); b.append(Consts.S.COMMA); b.append(_maxRespTime); b.append(w.getMax()); b.append(Consts.S.COMMA); - b.append(_minRespTime); b.append(w.getMin()); + b.append(_minRespTime); b.append(w.getMin()); b.append(Consts.S.COMMA); + + b.append(_2xxStatus); b.append(w.get2xxStatus()); b.append(Consts.S.COMMA); + b.append(_4xxStatus); b.append(w.get4xxStatus()); b.append(Consts.S.COMMA); + b.append(_5xxStatus); b.append(w.get5xxStatus()); b.append(Consts.S.COMMA); + b.append(_504Status); b.append(w.get504Status()); + b.append(Consts.S.RIGHT_BRACE); String msg = b.toString(); if ("kafka".equals(flowStatSchedConfigProperties.getDest())) { // for internal use diff --git a/fizz-core/src/main/java/we/config/SystemConfig.java b/fizz-core/src/main/java/we/config/SystemConfig.java index eafd1ff..649542b 100644 --- a/fizz-core/src/main/java/we/config/SystemConfig.java +++ b/fizz-core/src/main/java/we/config/SystemConfig.java @@ -24,6 +24,7 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.cloud.context.config.annotation.RefreshScope; import org.springframework.stereotype.Component; import org.springframework.util.ObjectUtils; +import we.context.config.annotation.FizzRefreshScope; import we.util.Consts; import we.util.UUIDUtil; import we.util.WebUtils; @@ -37,7 +38,7 @@ import java.util.stream.Stream; * @author hongqiaowei */ -@RefreshScope +@FizzRefreshScope @Component public class SystemConfig { @@ -205,51 +206,51 @@ public class SystemConfig { appHeaders.clear(); appHeaders.add("fizz-appid"); for (String h : StringUtils.split(hdrs, ',')) { - appHeaders.add(h.trim()); + String trim = h.trim(); + if (trim.equals("fizz-appid")) { + continue; + } + appHeaders.add(trim); } } WebUtils.setAppHeaders(appHeaders); log.info("app headers: " + appHeaders); } - public List getAppHeaders() { - return appHeaders; - } - @Value("${custom.header.sign:}") public void setCustomSignHeaders(String hdrs) { if (StringUtils.isNotBlank(hdrs)) { signHeaders.clear(); signHeaders.add("fizz-sign"); for (String h : StringUtils.split(hdrs, ',')) { - signHeaders.add(h.trim()); + String trim = h.trim(); + if (trim.equals("fizz-sign")) { + continue; + } + signHeaders.add(trim); } } WebUtils.setSignHeaders(signHeaders); log.info("sign headers: " + signHeaders); } - public List getSignHeaders() { - return signHeaders; - } - @Value("${custom.header.ts:}") public void setCustomTimestampHeaders(String hdrs) { if (StringUtils.isNotBlank(hdrs)) { timestampHeaders.clear(); timestampHeaders.add("fizz-ts"); for (String h : StringUtils.split(hdrs, ',')) { - timestampHeaders.add(h.trim()); + String trim = h.trim(); + if (trim.equals("fizz-ts")) { + continue; + } + timestampHeaders.add(trim); } } WebUtils.setTimestampHeaders(timestampHeaders); log.info("timestamp headers: " + timestampHeaders); } - public List getTimestampHeaders() { - return timestampHeaders; - } - @Value("${aggregate-test-auth:false}") public void setAggregateTestAuth(boolean b) { aggregateTestAuth = b; @@ -303,7 +304,7 @@ public class SystemConfig { logHeaderSet.add(fizzTraceIdHeader); } WebUtils.LOG_HEADER_SET = logHeaderSet; - log.info("log header list: " + logHeaderSet.toString()); + log.info("log headers: " + logHeaderSet.toString()); } private void updateLogResponseBody(boolean newValue) { diff --git a/fizz-core/src/main/java/we/context/config/annotation/FizzRefreshScope.java b/fizz-core/src/main/java/we/context/config/annotation/FizzRefreshScope.java new file mode 100644 index 0000000..9d924c2 --- /dev/null +++ b/fizz-core/src/main/java/we/context/config/annotation/FizzRefreshScope.java @@ -0,0 +1,37 @@ +/* + * Copyright (C) 2020 the original author or authors. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package we.context.config.annotation; + +import org.springframework.context.annotation.Scope; +import org.springframework.context.annotation.ScopedProxyMode; +import we.config.FizzConfigConfiguration; + +import java.lang.annotation.*; + +/** + * @author hongqiaowei + */ + +@Target({ElementType.TYPE, ElementType.METHOD}) +@Retention(RetentionPolicy.RUNTIME) +@Scope(FizzConfigConfiguration.REFRESH_SCOPE_NAME) +@Documented +public @interface FizzRefreshScope { + + ScopedProxyMode proxyMode() default ScopedProxyMode.TARGET_CLASS; +} diff --git a/fizz-core/src/main/java/we/context/event/FizzApplicationListener.java b/fizz-core/src/main/java/we/context/event/FizzApplicationListener.java new file mode 100644 index 0000000..aa7fec9 --- /dev/null +++ b/fizz-core/src/main/java/we/context/event/FizzApplicationListener.java @@ -0,0 +1,46 @@ +/* + * Copyright (C) 2021 the original author or authors. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package we.context.event; + +import org.springframework.context.ApplicationEvent; +import org.springframework.context.event.SmartApplicationListener; + +/** + * @author hongqiaowei + */ + +public class FizzApplicationListener implements SmartApplicationListener { + + // private static final Logger LOGGER = LoggerFactory.getLogger(FizzApplicationListener.class); + + @Override + public boolean supportsEventType(Class eventType) { + // return ApplicationPreparedEvent.class.isAssignableFrom(eventType); + return false; + } + + @Override + public void onApplicationEvent(ApplicationEvent event) { + /*if (event instanceof ApplicationPreparedEvent) { + ApplicationPreparedEvent evt = (ApplicationPreparedEvent) event; + ConfigurableEnvironment environment = evt.getApplicationContext().getEnvironment(); + if (environment instanceof StandardReactiveWebEnvironment) { + } + }*/ + } +} diff --git a/fizz-core/src/main/java/we/context/event/FizzRefreshEvent.java b/fizz-core/src/main/java/we/context/event/FizzRefreshEvent.java new file mode 100644 index 0000000..12c7995 --- /dev/null +++ b/fizz-core/src/main/java/we/context/event/FizzRefreshEvent.java @@ -0,0 +1,53 @@ +/* + * Copyright (C) 2021 the original author or authors. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package we.context.event; + +import org.springframework.context.ApplicationEvent; +import we.util.JacksonUtils; + +/** + * @author hongqiaowei + */ + +public class FizzRefreshEvent extends ApplicationEvent { + + public static final byte ENV_CHANGE = 1; + + private final byte type; + + private final Object data; + + public FizzRefreshEvent(Object source, byte type, Object data) { + super(source); + this.type = type; + this.data = data; + } + + public byte getType() { + return this.type; + } + + public Object getData() { + return this.data; + } + + @Override + public String toString() { + return JacksonUtils.writeValueAsString(this); + } +} diff --git a/fizz-core/src/main/java/we/context/event/FizzRefreshEventListener.java b/fizz-core/src/main/java/we/context/event/FizzRefreshEventListener.java new file mode 100644 index 0000000..a196ab4 --- /dev/null +++ b/fizz-core/src/main/java/we/context/event/FizzRefreshEventListener.java @@ -0,0 +1,104 @@ +/* + * Copyright (C) 2021 the original author or authors. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package we.context.event; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.ApplicationEvent; +import org.springframework.context.event.SmartApplicationListener; +import we.beans.factory.config.FizzBeanFactoryPostProcessor; +import we.context.scope.refresh.FizzRefreshScope; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * @author hongqiaowei + */ + +public class FizzRefreshEventListener implements SmartApplicationListener { + + private static final Logger LOGGER = LoggerFactory.getLogger(FizzRefreshEventListener.class); + + private final FizzBeanFactoryPostProcessor fizzBeanFactoryPostProcessor; + + private final FizzRefreshScope fizzRefreshScope; + + private final AtomicBoolean ready = new AtomicBoolean(false); + + public FizzRefreshEventListener(FizzBeanFactoryPostProcessor fizzBeanFactoryPostProcessor, FizzRefreshScope fizzRefreshScope) { + this.fizzBeanFactoryPostProcessor = fizzBeanFactoryPostProcessor; + this.fizzRefreshScope = fizzRefreshScope; + } + + @Override + public boolean supportsEventType(Class eventType) { + return ApplicationReadyEvent.class.isAssignableFrom(eventType) || FizzRefreshEvent.class.isAssignableFrom(eventType); + } + + @Override + public void onApplicationEvent(ApplicationEvent event) { + if (event instanceof ApplicationReadyEvent) { + handle((ApplicationReadyEvent) event); + } else if (event instanceof FizzRefreshEvent) { + handle((FizzRefreshEvent) event); + } + } + + public void handle(ApplicationReadyEvent event) { + this.ready.compareAndSet(false, true); + } + + public void handle(FizzRefreshEvent event) { + if (this.ready.get()) { + // EnvironmentChangeEvent ? + if (event.getType() == FizzRefreshEvent.ENV_CHANGE) { + /*Map> bean2propertyValuesMap = new HashMap<>(); + Map changedPropertyValueMap = (Map) event.getData(); + changedPropertyValueMap.forEach( + (property, value) -> { + String bean = fizzBeanFactoryPostProcessor.getBean(property); + if (bean != null) { + Map propertyValueMap = bean2propertyValuesMap.computeIfAbsent(bean, k -> new HashMap<>()); + propertyValueMap.put(property, value); + } + } + ); + bean2propertyValuesMap.forEach( + (bean, propertyValueMap) -> { + fizzRefreshScope.refresh(bean); + LOGGER.info("fizz refresh {} bean with {}", bean, propertyValueMap); + } + );*/ + Map changedPropertyValue = (Map) event.getData(); + changedPropertyValue.forEach( + (property, value) -> { + String bean = fizzBeanFactoryPostProcessor.getBean(property); + if (bean != null) { + fizzRefreshScope.refresh(bean); + LOGGER.info("fizz refresh {} bean with {}={}", bean, property, value); + } + } + ); + } + } + } + +} diff --git a/fizz-core/src/main/java/we/context/scope/refresh/FizzRefreshScope.java b/fizz-core/src/main/java/we/context/scope/refresh/FizzRefreshScope.java new file mode 100644 index 0000000..323dfd5 --- /dev/null +++ b/fizz-core/src/main/java/we/context/scope/refresh/FizzRefreshScope.java @@ -0,0 +1,121 @@ +/* + * Copyright (C) 2021 the original author or authors. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package we.context.scope.refresh; + +import org.springframework.beans.BeansException; +import org.springframework.beans.factory.config.BeanDefinition; +import org.springframework.beans.factory.support.BeanDefinitionRegistry; +import org.springframework.cloud.context.scope.GenericScope; +import org.springframework.context.ApplicationContext; +import org.springframework.context.ApplicationContextAware; +import org.springframework.context.ApplicationListener; +import org.springframework.context.event.ContextRefreshedEvent; +import org.springframework.core.Ordered; +import org.springframework.jmx.export.annotation.ManagedOperation; +import org.springframework.jmx.export.annotation.ManagedResource; +import we.config.FizzConfigConfiguration; + +/** + * @author hongqiaowei + */ + +@ManagedResource +public class FizzRefreshScope extends GenericScope implements ApplicationContextAware, + ApplicationListener, Ordered { + + private ApplicationContext context; + + private BeanDefinitionRegistry registry; + + private boolean eager = true; + + private int order = Ordered.LOWEST_PRECEDENCE - 100; + + public FizzRefreshScope() { + super.setName(FizzConfigConfiguration.REFRESH_SCOPE_NAME); + } + + @Override + public int getOrder() { + return this.order; + } + + public void setOrder(int order) { + this.order = order; + } + + public void setEager(boolean eager) { + this.eager = eager; + } + + @Override + public void postProcessBeanDefinitionRegistry(BeanDefinitionRegistry registry) + throws BeansException { + this.registry = registry; + super.postProcessBeanDefinitionRegistry(registry); + } + + @Override + public void onApplicationEvent(ContextRefreshedEvent event) { + start(event); + } + + public void start(ContextRefreshedEvent event) { + if (event.getApplicationContext() == this.context && this.eager + && this.registry != null) { + eagerlyInitialize(); + } + } + + private void eagerlyInitialize() { + for (String name : this.context.getBeanDefinitionNames()) { + BeanDefinition definition = this.registry.getBeanDefinition(name); + if (this.getName().equals(definition.getScope()) + && !definition.isLazyInit()) { + Object bean = this.context.getBean(name); + if (bean != null) { + bean.getClass(); + } + } + } + } + + @ManagedOperation(description = "Dispose of the current instance of bean name provided and force a refresh on next method execution.") + public boolean refresh(String name) { + if (!name.startsWith(SCOPED_TARGET_PREFIX)) { + name = SCOPED_TARGET_PREFIX + name; + } + if (super.destroy(name)) { + this.context.publishEvent(new FizzRefreshScopeRefreshedEvent(name)); + return true; + } + return false; + } + + @ManagedOperation(description = "Dispose of the current instance of all beans in this scope and force a refresh on next method execution.") + public void refreshAll() { + super.destroy(); + this.context.publishEvent(new FizzRefreshScopeRefreshedEvent()); + } + + @Override + public void setApplicationContext(ApplicationContext context) throws BeansException { + this.context = context; + } + +} diff --git a/fizz-core/src/main/java/we/context/scope/refresh/FizzRefreshScopeRefreshedEvent.java b/fizz-core/src/main/java/we/context/scope/refresh/FizzRefreshScopeRefreshedEvent.java new file mode 100644 index 0000000..47b0d23 --- /dev/null +++ b/fizz-core/src/main/java/we/context/scope/refresh/FizzRefreshScopeRefreshedEvent.java @@ -0,0 +1,44 @@ +/* + * Copyright (C) 2021 the original author or authors. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package we.context.scope.refresh; + +import org.springframework.context.ApplicationEvent; + +/** + * @author hongqiaowei + */ + +public class FizzRefreshScopeRefreshedEvent extends ApplicationEvent { + + public static final String DEFAULT_NAME = "__refreshAll__"; // TODO + + private String name; + + public FizzRefreshScopeRefreshedEvent() { + this(DEFAULT_NAME); + } + + public FizzRefreshScopeRefreshedEvent(String name) { + super(name); + this.name = name; + } + + public String getName() { + return this.name; + } +} diff --git a/fizz-core/src/main/java/we/controller/ManagerConfigController.java b/fizz-core/src/main/java/we/controller/ManagerConfigController.java index 5608b5e..13b9d44 100644 --- a/fizz-core/src/main/java/we/controller/ManagerConfigController.java +++ b/fizz-core/src/main/java/we/controller/ManagerConfigController.java @@ -17,8 +17,11 @@ package we.controller; +import com.google.common.collect.Sets; import org.springframework.beans.factory.annotation.Value; import org.springframework.cloud.context.config.annotation.RefreshScope; +import org.springframework.http.HttpMethod; +import org.springframework.util.CollectionUtils; import org.springframework.util.StringUtils; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; @@ -26,15 +29,29 @@ import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; import reactor.core.publisher.Mono; import we.controller.req.BaseManagerConfigReq; +import we.controller.req.GetApiConfigDetailReq; +import we.controller.req.GetApiConfigReq; import we.controller.req.GetConfigReq; import we.controller.req.GetConfigStrReq; +import we.controller.resp.ApiConfigInfo; import we.controller.resp.ConfigResp; import we.controller.resp.ConfigStrResp; +import we.controller.resp.GetApiConfigDetailResp; +import we.controller.resp.GetApiConfigResp; import we.fizz.ConfigLoader; +import we.plugin.PluginConfig; +import we.plugin.auth.ApiConfig; +import we.plugin.auth.ApiConfig2appsService; +import we.plugin.auth.ApiConfigService; +import we.plugin.auth.GatewayGroup; +import we.plugin.auth.GatewayGroupService; import javax.annotation.Resource; import javax.validation.Valid; +import java.util.ArrayList; +import java.util.Comparator; import java.util.List; +import java.util.Set; import java.util.stream.Collectors; /** @@ -46,12 +63,27 @@ import java.util.stream.Collectors; @RestController @RequestMapping(value = "/admin/managerConfig") public class ManagerConfigController { + /** + * 路由管理的路由类型集合 + */ + public static final Set API_AUTH_PROXY_MODE_SET = Sets.newHashSet(ApiConfig.Type.SERVICE_AGGREGATE, + ApiConfig.Type.SERVICE_DISCOVERY, ApiConfig.Type.REVERSE_PROXY, ApiConfig.Type.DUBBO); + @Value("${fizz.manager.config.key:fizz-manager-key}") private String key; @Resource private ConfigLoader configLoader; + @Resource + private ApiConfigService apiConfigService; + + @Resource + private GatewayGroupService gatewayGroupService; + + @Resource + private ApiConfig2appsService apiConfig2appsService; + @PostMapping("/getAggregateConfigs") public Mono getAggregateConfigs(@Valid @RequestBody GetConfigReq getConfigReq) { this.checkSign(getConfigReq); @@ -93,6 +125,189 @@ public class ManagerConfigController { return Mono.just(configStrResp); } + @PostMapping("/getApiConfigs") + public Mono getApiConfigs(@Valid @RequestBody GetApiConfigReq getApiConfigReq) { + this.checkSign(getApiConfigReq); + Integer current = getApiConfigReq.getCurrent(); + Integer size = getApiConfigReq.getSize(); + + String gatewayGroup = getApiConfigReq.getGatewayGroup(); + String service = getApiConfigReq.getService(); + String path = getApiConfigReq.getPath(); + Set methods = getApiConfigReq.getMethods(); + Set plugins = getApiConfigReq.getPlugins(); + String access = getApiConfigReq.getAccess(); + + Set currentGatewayGroupSet = gatewayGroupService.currentGatewayGroupSet; + + List apiConfigInfoList = apiConfigService.getApiConfigMap().values().stream().filter(it -> { + if (!currentGatewayGroupSet.contains(it.firstGatewayGroup)) { + return false; + } + + if (!API_AUTH_PROXY_MODE_SET.contains(it.type)) { + return false; + } + + if (StringUtils.hasText(gatewayGroup)) { + if (!it.firstGatewayGroup.equals(gatewayGroup)) { + return false; + } + } + + if (StringUtils.hasText(service)) { + if (!it.service.contains(service)) { + return false; + } + } + + if (StringUtils.hasText(path)) { + if (!it.path.contains(path)) { + return false; + } + } + + if (!CollectionUtils.isEmpty(methods)) { + if (!methods.contains(it.fizzMethod instanceof HttpMethod ? ((HttpMethod) it.fizzMethod).name() : it.fizzMethod)) { + return false; + } + } + + if (!CollectionUtils.isEmpty(plugins)) { + boolean match = false; + for (PluginConfig pluginConfig : it.pluginConfigs) { + if (plugins.contains(pluginConfig.plugin)) { + match = true; + break; + } + } + + if (!match) { + GatewayGroup group = gatewayGroupService.get(it.firstGatewayGroup); + if (group != null) { + for (PluginConfig pluginConfig : group.pluginConfigs) { + if (plugins.contains(pluginConfig.plugin)) { + match = true; + break; + } + } + } + + if (!match) { + return false; + } + } + } + + if ("a".equals(access) && !it.allowAccess) { + return false; + } + + if ("f".equals(access) && it.allowAccess) { + return false; + } + + return true; + }).map(it -> { + ApiConfigInfo apiConfigInfo = new ApiConfigInfo(); + apiConfigInfo.setId((long) it.id); + apiConfigInfo.setGatewayGroup(it.firstGatewayGroup); + apiConfigInfo.setProxyMode(it.type); + apiConfigInfo.setIsDedicatedLine(it.dedicatedLine ? (byte) 1 : 0); + apiConfigInfo.setService(it.service); + apiConfigInfo.setPath(it.path); + apiConfigInfo.setMethod(it.fizzMethod instanceof HttpMethod ? ((HttpMethod) it.fizzMethod).name() : ""); + + List gatewayGroupPluginConfigs = null; + GatewayGroup group = gatewayGroupService.get(it.firstGatewayGroup); + if (group != null) { + gatewayGroupPluginConfigs = group.pluginConfigs; + } + if (CollectionUtils.isEmpty(gatewayGroupPluginConfigs)) { + apiConfigInfo.setPlugins(it.pluginConfigs.stream().map(pluginConfig -> pluginConfig.plugin).collect(Collectors.toList())); + } else { + List pcs = new ArrayList<>(gatewayGroupPluginConfigs.size() + it.pluginConfigs.size()); + pcs.addAll(gatewayGroupPluginConfigs); + pcs.addAll(it.pluginConfigs); + pcs.sort(null); + apiConfigInfo.setPlugins(pcs.stream().map(pluginConfig -> pluginConfig.plugin).collect(Collectors.toList())); + } + + apiConfigInfo.setAccess(it.allowAccess ? "a" : "f"); + return apiConfigInfo; + }).collect(Collectors.toList()); + + GetApiConfigResp getApiConfigResp = new GetApiConfigResp(); + getApiConfigResp.setTotal((long) apiConfigInfoList.size()); + + apiConfigInfoList.sort(Comparator.comparing(ApiConfigInfo::getId).reversed()); + + int apiConfigListSize = apiConfigInfoList.size(); + int startIndex = (current - 1) * size; + if (startIndex >= apiConfigListSize) { + return Mono.just(getApiConfigResp); + } + + int endIndex = current * size; + if (endIndex > apiConfigListSize) { + endIndex = apiConfigListSize; + } + getApiConfigResp.setApiConfigInfos(apiConfigInfoList.subList(startIndex, endIndex)); + return Mono.just(getApiConfigResp); + } + + + @PostMapping("/getApiConfigDetail") + public Mono getApiConfigDetail(@Valid @RequestBody GetApiConfigDetailReq getApiConfigDetailReq) { + this.checkSign(getApiConfigDetailReq); + Long apiConfigId = getApiConfigDetailReq.getApiConfigId(); + + ApiConfig apiConfig = apiConfigService.getApiConfigMap().get(apiConfigId.intValue()); + + GetApiConfigDetailResp getApiConfigDetailResp = new GetApiConfigDetailResp(); + if (apiConfig == null) { + getApiConfigDetailResp.setExist(false); + } else { + getApiConfigDetailResp.setExist(true); + getApiConfigDetailResp.setId((long) apiConfig.id); + getApiConfigDetailResp.setGatewayGroup(apiConfig.firstGatewayGroup); + getApiConfigDetailResp.setService(apiConfig.service); + getApiConfigDetailResp.setMethod(apiConfig.fizzMethod instanceof HttpMethod ? ((HttpMethod) apiConfig.fizzMethod).name() : ""); + getApiConfigDetailResp.setPath(apiConfig.path); + getApiConfigDetailResp.setAppEnabled(apiConfig.checkApp); + if (apiConfig.checkApp) { + getApiConfigDetailResp.setApps(apiConfig2appsService.getApiConfig2appsMap().get(apiConfig.id)); + } + getApiConfigDetailResp.setAccess(apiConfig.allowAccess ? "a" : "f"); + getApiConfigDetailResp.setProxyMode(apiConfig.type); + getApiConfigDetailResp.setBackendService(apiConfig.backendService); + getApiConfigDetailResp.setBackendPath(apiConfig.backendPath); + + getApiConfigDetailResp.setApiPlugins(apiConfig.pluginConfigs.stream().map(pluginConfig -> { + GetApiConfigDetailResp.ApiPluginVO apiPluginVO = new GetApiConfigDetailResp.ApiPluginVO(); + apiPluginVO.setPlugin(pluginConfig.plugin); + apiPluginVO.setConfig(pluginConfig.config); + apiPluginVO.setOrder(pluginConfig.order); + return apiPluginVO; + }).collect(Collectors.toList())); + + getApiConfigDetailResp.setApiBackends(apiConfig.httpHostPorts); + getApiConfigDetailResp.setRpcMethod(apiConfig.rpcMethod); + getApiConfigDetailResp.setRpcParamTypes(apiConfig.rpcParamTypes); + getApiConfigDetailResp.setRpcVersion(apiConfig.rpcVersion); + getApiConfigDetailResp.setRpcGroup(apiConfig.rpcGroup); + getApiConfigDetailResp.setTimeout((int) apiConfig.timeout); + getApiConfigDetailResp.setRetryCount(apiConfig.retryCount); + getApiConfigDetailResp.setRetryInterval(apiConfig.retryInterval); + + getApiConfigDetailResp.setIsDedicatedLine(apiConfig.dedicatedLine ? (byte) 1 : 0); + + getApiConfigDetailResp.setRegistryName(apiConfig.registryCenter); + } + + return Mono.just(getApiConfigDetailResp); + } + private void checkSign(BaseManagerConfigReq req) { if (!req.checkSign(key)) { throw new RuntimeException("验证签名失败"); diff --git a/fizz-core/src/main/java/we/controller/req/GetApiConfigDetailReq.java b/fizz-core/src/main/java/we/controller/req/GetApiConfigDetailReq.java new file mode 100644 index 0000000..b78df71 --- /dev/null +++ b/fizz-core/src/main/java/we/controller/req/GetApiConfigDetailReq.java @@ -0,0 +1,41 @@ +/* + * Copyright (C) 2020 the original author or authors. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package we.controller.req; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.springframework.util.DigestUtils; + +/** + * Get api config detail request entity + * + * @author zhongjie + * @since 2.6.0 + */ +@Data +@EqualsAndHashCode(callSuper = true) +public class GetApiConfigDetailReq extends BaseManagerConfigReq { + /** + * Api config ID + */ + private Long apiConfigId; + + @Override + boolean innerCheckSign(String key, String sign, Long timestamp) { + return DigestUtils.md5DigestAsHex(String.format("%s-%s-%s", apiConfigId, timestamp, key).getBytes()).equals(sign); + } +} diff --git a/fizz-core/src/main/java/we/controller/req/GetApiConfigReq.java b/fizz-core/src/main/java/we/controller/req/GetApiConfigReq.java new file mode 100644 index 0000000..0609433 --- /dev/null +++ b/fizz-core/src/main/java/we/controller/req/GetApiConfigReq.java @@ -0,0 +1,72 @@ +/* + * Copyright (C) 2020 the original author or authors. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package we.controller.req; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.springframework.util.DigestUtils; + +import java.util.Set; + +/** + * Get api config request entity + * + * @author zhongjie + * @since 2.6.0 + */ +@Data +@EqualsAndHashCode(callSuper = true) +public class GetApiConfigReq extends BaseManagerConfigReq { + /** + * Gateway group + */ + private String gatewayGroup; + /** + * Frontend service name + */ + private String service; + /** + * Frontend API path + */ + private String path; + /** + * Methods + */ + private Set methods; + /** + * Plugin english names + */ + private Set plugins; + /** + * Can access ? a-allow，f-forbid + */ + private String access; + + /** + * Current index + */ + private Integer current; + /** + * Size of page + */ + private Integer size; + + @Override + boolean innerCheckSign(String key, String sign, Long timestamp) { + return DigestUtils.md5DigestAsHex(String.format("%s-%s", timestamp, key).getBytes()).equals(sign); + } +} diff --git a/fizz-core/src/main/java/we/controller/resp/ApiConfigInfo.java b/fizz-core/src/main/java/we/controller/resp/ApiConfigInfo.java new file mode 100644 index 0000000..294de37 --- /dev/null +++ b/fizz-core/src/main/java/we/controller/resp/ApiConfigInfo.java @@ -0,0 +1,69 @@ +/* + * Copyright (C) 2020 the original author or authors. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package we.controller.resp; + +import lombok.Data; + +import java.io.Serializable; +import java.util.List; + +/** + * Api config entity + * + * @author zhongjie + * @since 2.6.0 + */ +@Data +public class ApiConfigInfo implements Serializable { + private static final long serialVersionUID = 1L; + /** + * Identifier + */ + private Long id; + /** + * Gateway group + */ + private String gatewayGroup; + /** + * Proxy mode: 1-aggregate 2-discovery 3-proxy 4-callback 5-Dubbo + */ + private Byte proxyMode; + /** + * Is dedicated line: 0-no 1-yes + */ + private Byte isDedicatedLine; + /** + * Frontend service name + */ + private String service; + /** + * Frontend API path + */ + private String path; + /** + * Method + */ + private String method; + /** + * Can access ? a-allow，f-forbid + */ + private String access; + /** + * Plugin english names + */ + private List plugins; +} \ No newline at end of file diff --git a/fizz-core/src/main/java/we/controller/resp/GetApiConfigDetailResp.java b/fizz-core/src/main/java/we/controller/resp/GetApiConfigDetailResp.java new file mode 100644 index 0000000..4067430 --- /dev/null +++ b/fizz-core/src/main/java/we/controller/resp/GetApiConfigDetailResp.java @@ -0,0 +1,155 @@ +/* + * Copyright (C) 2020 the original author or authors. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package we.controller.resp; + +import lombok.Data; + +import java.io.Serializable; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Get api config detail response entity + * + * @author zhongjie + * @since 2.6.0 + */ +@Data +public class GetApiConfigDetailResp implements Serializable { + private static final long serialVersionUID = 1L; + + /** + * Is exist + */ + private Boolean exist; + + /** + * ID + */ + private Long id; + /** + * Gateway group + */ + private String gatewayGroup; + /** + * Frontend service name + */ + private String service; + /** + * Method + */ + private String method; + /** + * Frontend API path + */ + private String path; + + /** + * Is app Enabled + */ + private Boolean appEnabled; + /** + * App set + */ + private Set apps; + + /** + * Can access ? a-allow, f-forbid + */ + private String access; + /** + * Proxy mode: 1-aggregate 2-discovery 3-proxy 4-callback 5-Dubbo + */ + private Byte proxyMode; + + /** + * Backend service name + */ + private String backendService; + /** + * Backend API path + */ + private String backendPath; + + /** + * Api plugin list + */ + private List apiPlugins; + + /** + * Api backend list + */ + private List apiBackends; + /** + * RPC method + */ + private String rpcMethod; + /** + * RPC parameter types + */ + private String rpcParamTypes; + /** + * RPC version + */ + private String rpcVersion; + /** + * RPC group + */ + private String rpcGroup; + /** + * Timeout millis + */ + private Integer timeout; + + /** + * Retry count + */ + private Integer retryCount; + /** + * Retry interval millis + */ + private Long retryInterval; + + /** + * Is dedicated line: 0-no 1-yes + */ + private Byte isDedicatedLine; + + /** + * Registry name + */ + private String registryName; + + @Data + public static class ApiPluginVO implements Serializable { + private static final long serialVersionUID = 1L; + + /** + * Plugin english name + */ + public String plugin; + /** + * Config map + */ + public Map config; + /** + * Order + */ + public Integer order; + } +} diff --git a/fizz-core/src/main/java/we/controller/resp/GetApiConfigResp.java b/fizz-core/src/main/java/we/controller/resp/GetApiConfigResp.java new file mode 100644 index 0000000..534d85a --- /dev/null +++ b/fizz-core/src/main/java/we/controller/resp/GetApiConfigResp.java @@ -0,0 +1,43 @@ +/* + * Copyright (C) 2020 the original author or authors. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package we.controller.resp; + +import lombok.Data; + +import java.io.Serializable; +import java.util.List; + +/** + * Get api config response entity + * + * @author zhongjie + * @since 2.6.0 + */ +@Data +public class GetApiConfigResp implements Serializable { + private static final long serialVersionUID = 1L; + + /** + * Api config infos + */ + private List apiConfigInfos; + + /** + * Total count + */ + private Long total; +} \ No newline at end of file diff --git a/fizz-core/src/main/java/we/filter/FilterExceptionHandlerConfig.java b/fizz-core/src/main/java/we/filter/FilterExceptionHandlerConfig.java index 19f4002..4cbc672 100644 --- a/fizz-core/src/main/java/we/filter/FilterExceptionHandlerConfig.java +++ b/fizz-core/src/main/java/we/filter/FilterExceptionHandlerConfig.java @@ -45,6 +45,7 @@ import we.util.ThreadContext; import we.util.WebUtils; import java.net.URI; +import java.util.concurrent.TimeoutException; /** * @author hongqiaowei @@ -60,6 +61,7 @@ public class FilterExceptionHandlerConfig { @Override public Mono handle(ServerWebExchange exchange, Throwable t) { + exchange.getAttributes().put(WebUtils.ORIGINAL_ERROR, t); String traceId = WebUtils.getTraceId(exchange); ServerHttpResponse resp = exchange.getResponse(); if (SystemConfig.FIZZ_ERR_RESP_HTTP_STATUS_ENABLE) { diff --git a/fizz-core/src/main/java/we/filter/FlowControlFilter.java b/fizz-core/src/main/java/we/filter/FlowControlFilter.java index f12a42b..2f767e6 100644 --- a/fizz-core/src/main/java/we/filter/FlowControlFilter.java +++ b/fizz-core/src/main/java/we/filter/FlowControlFilter.java @@ -24,6 +24,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.annotation.Order; import org.springframework.http.HttpHeaders; import org.springframework.http.HttpStatus; +import org.springframework.http.server.reactive.ServerHttpRequest; import org.springframework.http.server.reactive.ServerHttpResponse; import org.springframework.stereotype.Component; import org.springframework.web.server.ServerWebExchange; @@ -32,6 +33,7 @@ import reactor.core.publisher.Mono; import reactor.core.publisher.SignalType; import we.config.SystemConfig; import we.flume.clients.log4j2appender.LogService; +import we.monitor.FizzMonitorService; import we.plugin.auth.ApiConfigService; import we.plugin.auth.AppService; import we.stats.BlockType; @@ -48,6 +50,7 @@ import we.util.*; import javax.annotation.Resource; import java.util.ArrayList; import java.util.List; +import java.util.concurrent.TimeoutException; /** * @author hongqiaowei @@ -71,6 +74,10 @@ public class FlowControlFilter extends FizzWebFilter { private static final String _fizz = "_fizz-"; + private static final String concurrents = "concurrents"; + + private static final String qps = "qps"; + @Resource private FlowControlFilterProperties flowControlFilterProperties; @@ -96,10 +103,14 @@ public class FlowControlFilter extends FizzWebFilter { @Resource private CircuitBreakManager circuitBreakManager; + @Resource + private FizzMonitorService fizzMonitorService; + @Override public Mono doFilter(ServerWebExchange exchange, WebFilterChain chain) { - String path = exchange.getRequest().getPath().value(); + ServerHttpRequest request = exchange.getRequest(); + String path = request.getPath().value(); int secFS = path.indexOf(Consts.S.FORWARD_SLASH, 1); if (secFS == -1) { return WebUtils.responseError(exchange, HttpStatus.INTERNAL_SERVER_ERROR.value(), "request path should like /optional-prefix/service-name/real-biz-path"); @@ -133,15 +144,18 @@ public class FlowControlFilter extends FizzWebFilter { String ip = WebUtils.getOriginIp(exchange); long currentTimeSlot = flowStat.currentTimeSlotId(); - List resourceConfigs = getFlowControlConfigs(app, ip, null, service, path); + String host = request.getHeaders().getFirst(HttpHeaders.HOST); + List resourceConfigs = getFlowControlConfigs(app, ip, host, service, path); IncrRequestResult result = flowStat.incrRequest(exchange, resourceConfigs, currentTimeSlot, (rc, rcs) -> { return getResourceConfigItselfAndParents(rc, rcs); }); if (result != null && !result.isSuccess()) { + long currentTimeMillis = System.currentTimeMillis(); String blockedResourceId = result.getBlockedResourceId(); if (BlockType.CIRCUIT_BREAK == result.getBlockType()) { - log.info("{} exceed {} circuit breaker limit", traceId, blockedResourceId, LogService.BIZ_ID, traceId); + fizzMonitorService.sendAlarm(service, path, FizzMonitorService.CIRCUIT_BREAK_ALARM, null, currentTimeMillis); + log.info("{} trigger {} circuit breaker limit", traceId, blockedResourceId, LogService.BIZ_ID, traceId); String responseContentType = flowControlFilterProperties.getDegradeDefaultResponseContentType(); String responseContent = flowControlFilterProperties.getDegradeDefaultResponseContent(); @@ -171,8 +185,10 @@ public class FlowControlFilter extends FizzWebFilter { } else { if (BlockType.CONCURRENT_REQUEST == result.getBlockType()) { + fizzMonitorService.sendAlarm(service, path, FizzMonitorService.RATE_LIMIT_ALARM, concurrents, currentTimeMillis); log.info("{} exceed {} flow limit, blocked by maximum concurrent requests", traceId, blockedResourceId, LogService.BIZ_ID, traceId); } else { + fizzMonitorService.sendAlarm(service, path, FizzMonitorService.RATE_LIMIT_ALARM, qps, currentTimeMillis); log.info("{} exceed {} flow limit, blocked by maximum QPS", traceId, blockedResourceId, LogService.BIZ_ID, traceId); } @@ -201,13 +217,25 @@ public class FlowControlFilter extends FizzWebFilter { return chain.filter(exchange).doFinally(s -> { long rt = System.currentTimeMillis() - start; CircuitBreaker cb = exchange.getAttribute(CircuitBreaker.DETECT_REQUEST); - if (s == SignalType.ON_ERROR || exchange.getResponse().getStatusCode().is5xxServerError()) { - flowStat.addRequestRT(resourceConfigs, currentTimeSlot, rt, false); + HttpStatus statusCode = exchange.getResponse().getStatusCode(); + Throwable t = exchange.getAttribute(WebUtils.ORIGINAL_ERROR); + if (t instanceof TimeoutException) { + statusCode = HttpStatus.GATEWAY_TIMEOUT; + } + if (s == SignalType.ON_ERROR || statusCode.is4xxClientError() || statusCode.is5xxServerError()) { + flowStat.addRequestRT(resourceConfigs, currentTimeSlot, rt, false, statusCode); if (cb != null) { cb.transit(CircuitBreaker.State.RESUME_DETECTIVE, CircuitBreaker.State.OPEN, currentTimeSlot, flowStat); } + if (statusCode == HttpStatus.GATEWAY_TIMEOUT) { + fizzMonitorService.sendAlarm(finalService, finalPath, FizzMonitorService.TIMEOUT_ALARM, t.getMessage(), start); + } else if (statusCode.is5xxServerError()) { + fizzMonitorService.sendAlarm(finalService, finalPath, FizzMonitorService.ERROR_ALARM, String.valueOf(statusCode.value()), start); + } else if (s == SignalType.ON_ERROR && t != null) { + fizzMonitorService.sendAlarm(finalService, finalPath, FizzMonitorService.ERROR_ALARM, t.getMessage(), start); + } } else { - flowStat.addRequestRT(resourceConfigs, currentTimeSlot, rt, true); + flowStat.addRequestRT(resourceConfigs, currentTimeSlot, rt, true, statusCode); if (cb != null) { cb.transit(CircuitBreaker.State.RESUME_DETECTIVE, CircuitBreaker.State.CLOSED, currentTimeSlot, flowStat); } @@ -248,6 +276,11 @@ public class FlowControlFilter extends FizzWebFilter { for (int i = rcs.size() - 1; i > -1; i--) { ResourceConfig r = rcs.get(i); String id = r.getResourceId(); + String node = ResourceIdUtils.getNode(id); + if (node != null && !node.equals(ResourceIdUtils.NODE)) { + result.add(r); + continue; + } String app = ResourceIdUtils.getApp(id); String ip = ResourceIdUtils.getIp(id); String path = ResourceIdUtils.getPath(id); @@ -286,10 +319,17 @@ public class FlowControlFilter extends FizzWebFilter { if (log.isDebugEnabled()) { log.debug("get flow control configs by app={}, ip={}, node={}, service={}, path={}", app, ip, node, service, path); } - List resourceConfigs = new ArrayList<>(9); + boolean hasHost = (StringUtils.isNotBlank(node) && !node.equals(ResourceIdUtils.NODE)); + int sz = hasHost ? 10 : 9; + List resourceConfigs = new ArrayList<>(sz); StringBuilder b = ThreadContext.getStringBuilder(); checkRateLimitConfigAndAddTo(resourceConfigs, b, null, null, ResourceIdUtils.NODE, null, null, null); + if (hasHost) { + String resourceId = ResourceIdUtils.buildResourceId(app, ip, node, service, path); + ResourceConfig resourceConfig = new ResourceConfig(resourceId, 0, 0); + resourceConfigs.add(resourceConfig); + } checkRateLimitConfigAndAddTo(resourceConfigs, b, null, null, null, service, null, ResourceIdUtils.SERVICE_DEFAULT); checkRateLimitConfigAndAddTo(resourceConfigs, b, null, null, null, service, path, null); diff --git a/fizz-core/src/main/java/we/monitor/FizzMonitorService.java b/fizz-core/src/main/java/we/monitor/FizzMonitorService.java new file mode 100644 index 0000000..da4693b --- /dev/null +++ b/fizz-core/src/main/java/we/monitor/FizzMonitorService.java @@ -0,0 +1,91 @@ +/* + * Copyright (C) 2020 the original author or authors. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package we.monitor; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.data.redis.core.ReactiveStringRedisTemplate; +import org.springframework.stereotype.Service; +import we.config.AggregateRedisConfig; +import we.flume.clients.log4j2appender.LogService; +import we.util.Consts; +import we.util.ThreadContext; + +import javax.annotation.Resource; + +/** + * @author hongqiaowei + */ + +@Service +public class FizzMonitorService { + + private static final Logger LOGGER = LoggerFactory.getLogger(FizzMonitorService.class); + + public static final byte ERROR_ALARM = 1; + public static final byte TIMEOUT_ALARM = 2; + public static final byte RATE_LIMIT_ALARM = 3; + public static final byte CIRCUIT_BREAK_ALARM = 4; + + private static final String _service = "\"service\":"; + private static final String _path = "\"path\":"; + private static final String _type = "\"type\":"; + private static final String _desc = "\"desc\":"; + private static final String _timestamp = "\"timestamp\":"; + + @Value("${fizz.monitor.alarm.enable:true}") + private boolean alarmEnable; + + @Value("${fizz.monitor.alarm.dest:redis}") + private String dest; + + @Value("${fizz.monitor.alarm.queue:fizz_alarm_channel}") + private String queue; + + @Resource(name = AggregateRedisConfig.AGGREGATE_REACTIVE_REDIS_TEMPLATE) + private ReactiveStringRedisTemplate rt; + + public void sendAlarm(String service, String path, byte type, String desc, long timestamp) { + if (alarmEnable) { + StringBuilder b = ThreadContext.getStringBuilder(); + b.append(Consts.S.LEFT_BRACE); + b.append(_service); toJsonStrVal(b, service); b.append(Consts.S.COMMA); + b.append(_path); toJsonStrVal(b, path); b.append(Consts.S.COMMA); + b.append(_type); b.append(type); b.append(Consts.S.COMMA); + + if (desc != null) { + b.append(_desc); toJsonStrVal(b, desc); b.append(Consts.S.COMMA); + } + + b.append(_timestamp) .append(timestamp); + b.append(Consts.S.RIGHT_BRACE); + String msg = b.toString(); + if (Consts.KAFKA.equals(dest)) { // for internal use + LOGGER.warn(msg, LogService.HANDLE_STGY, LogService.toKF(queue)); + } else { + rt.convertAndSend(queue, msg).subscribe(); + } + } + } + + private static void toJsonStrVal(StringBuilder b, String value) { + b.append(Consts.S.DOUBLE_QUOTE).append(value).append(Consts.S.DOUBLE_QUOTE); + } + +} diff --git a/fizz-core/src/main/java/we/plugin/auth/ApiConfigService.java b/fizz-core/src/main/java/we/plugin/auth/ApiConfigService.java index 50fd0ba..e785603 100644 --- a/fizz-core/src/main/java/we/plugin/auth/ApiConfigService.java +++ b/fizz-core/src/main/java/we/plugin/auth/ApiConfigService.java @@ -317,6 +317,10 @@ public class ApiConfigService implements ApplicationListener getApiConfigMap() { + return apiConfigMap; + } + /** * @deprecated */ diff --git a/fizz-core/src/main/java/we/plugin/stat/StatPluginFilterProperties.java b/fizz-core/src/main/java/we/plugin/stat/StatPluginFilterProperties.java index e09820c..cd8b471 100644 --- a/fizz-core/src/main/java/we/plugin/stat/StatPluginFilterProperties.java +++ b/fizz-core/src/main/java/we/plugin/stat/StatPluginFilterProperties.java @@ -21,6 +21,7 @@ import lombok.Data; import org.springframework.beans.factory.annotation.Value; import org.springframework.cloud.context.config.annotation.RefreshScope; import org.springframework.stereotype.Component; +import we.context.config.annotation.FizzRefreshScope; /** * {@link StatPluginFilter} properties @@ -28,7 +29,7 @@ import org.springframework.stereotype.Component; * @author zhongjie */ -@RefreshScope +@FizzRefreshScope @Component @Data public class StatPluginFilterProperties { diff --git a/fizz-core/src/main/java/we/proxy/dubbo/ApacheDubboGenericServiceProperties.java b/fizz-core/src/main/java/we/proxy/dubbo/ApacheDubboGenericServiceProperties.java index 27d413a..027b41c 100644 --- a/fizz-core/src/main/java/we/proxy/dubbo/ApacheDubboGenericServiceProperties.java +++ b/fizz-core/src/main/java/we/proxy/dubbo/ApacheDubboGenericServiceProperties.java @@ -20,13 +20,14 @@ import lombok.Data; import org.springframework.beans.factory.annotation.Value; import org.springframework.cloud.context.config.annotation.RefreshScope; import org.springframework.stereotype.Component; +import we.context.config.annotation.FizzRefreshScope; /** * {@link ApacheDubboGenericService} properties * * @author zhongjie */ -@RefreshScope +@FizzRefreshScope @Component @Data public class ApacheDubboGenericServiceProperties { diff --git a/fizz-core/src/main/java/we/stats/FlowStat.java b/fizz-core/src/main/java/we/stats/FlowStat.java index 395a3bb..1ed724c 100644 --- a/fizz-core/src/main/java/we/stats/FlowStat.java +++ b/fizz-core/src/main/java/we/stats/FlowStat.java @@ -33,6 +33,7 @@ import java.util.function.BiFunction; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.http.HttpStatus; import org.springframework.web.server.ServerWebExchange; import we.stats.circuitbreaker.CircuitBreakManager; import we.stats.circuitbreaker.CircuitBreaker; @@ -293,7 +294,7 @@ public class FlowStat { * @param rt * @param isSuccess */ - public void addRequestRT(List resourceConfigs, long timeSlotId, long rt, boolean isSuccess) { + public void addRequestRT(List resourceConfigs, long timeSlotId, long rt, boolean isSuccess, HttpStatus statusCode) { if (resourceConfigs == null || resourceConfigs.size() == 0) { return; } @@ -301,10 +302,23 @@ public class FlowStat { ResourceStat resourceStat = getResourceStat(resourceConfigs.get(i).getResourceId()); resourceStat.decrConcurrentRequest(timeSlotId); resourceStat.addRequestRT(timeSlotId, rt, isSuccess); + + if (statusCode.is2xxSuccessful()) { + resourceStat.incr2xxStatusCount(timeSlotId); + + } else if (statusCode.is4xxClientError()) { + resourceStat.incr4xxStatusCount(timeSlotId); + + } else if (statusCode.is5xxServerError()) { + resourceStat.incr5xxStatusCount(timeSlotId); + + } else if (statusCode == HttpStatus.GATEWAY_TIMEOUT) { + resourceStat.incr504StatusCount(timeSlotId); + } } } - public void addRequestRT(ServerWebExchange exchange, List resourceConfigs, long timeSlotId, long rt, boolean isSuccess) { + /*public void addRequestRT(ServerWebExchange exchange, List resourceConfigs, long timeSlotId, long rt, boolean isSuccess) { if (resourceConfigs == null || resourceConfigs.size() == 0) { return; } @@ -317,7 +331,7 @@ public class FlowStat { String service = WebUtils.getClientService(exchange); String path = WebUtils.getClientReqPath(exchange); circuitBreakManager.correctCircuitBreakerStateAsError(exchange, timeSlotId, this, service, path); - } + }*/ /** * Increase concurrent request counter of the specified resource diff --git a/fizz-core/src/main/java/we/stats/ResourceStat.java b/fizz-core/src/main/java/we/stats/ResourceStat.java index 2f2834c..0c3ea81 100644 --- a/fizz-core/src/main/java/we/stats/ResourceStat.java +++ b/fizz-core/src/main/java/we/stats/ResourceStat.java @@ -202,6 +202,22 @@ public class ResourceStat { getTimeSlot(timeSlot).getGradualRejectNum().decrementAndGet(); } + public void incr2xxStatusCount(long timeSlot) { + getTimeSlot(timeSlot).get2xxStatusCount().incrementAndGet(); + } + + public void incr4xxStatusCount(long timeSlot) { + getTimeSlot(timeSlot).get4xxStatusCount().incrementAndGet(); + } + + public void incr5xxStatusCount(long timeSlot) { + getTimeSlot(timeSlot).get5xxStatusCount().incrementAndGet(); + } + + public void incr504StatusCount(long timeSlot) { + getTimeSlot(timeSlot).get504StatusCount().incrementAndGet(); + } + /** * Add request RT to the specified time slot * @@ -237,6 +253,12 @@ public class ResourceStat { long blockReqs = 0; long totalBlockReqs = 0; long compReqs = 0; + + int _2xxStatus = 0; + int _4xxStatus = 0; + int _5xxStatus = 0; + int _504Status = 0; + for (long i = startSlotId; i < endSlotId;) { if (timeSlots.containsKey(i)) { TimeSlot timeSlot = timeSlots.get(i); @@ -252,6 +274,11 @@ public class ResourceStat { blockReqs = blockReqs + timeSlot.getBlockRequests().get(); totalBlockReqs = totalBlockReqs + timeSlot.getTotalBlockRequests().get(); compReqs = compReqs + timeSlot.getCompReqs().get(); + + _2xxStatus = _2xxStatus + timeSlot.get2xxStatusCount().get(); + _4xxStatus = _4xxStatus + timeSlot.get4xxStatusCount().get(); + _5xxStatus = _5xxStatus + timeSlot.get5xxStatusCount().get(); + _504Status = _504Status + timeSlot.get504StatusCount().get(); } i = i + FlowStat.INTERVAL; } @@ -265,6 +292,11 @@ public class ResourceStat { tws.setCompReqs(compReqs); tws.setPeakRps(new BigDecimal(peakRps)); + tws.set2xxStatus(_2xxStatus); + tws.set4xxStatus(_4xxStatus); + tws.set5xxStatus(_5xxStatus); + tws.set504Status(_504Status); + if (compReqs > 0) { tws.setAvgRt(totalRt / compReqs); } @@ -308,5 +340,4 @@ public class ResourceStat { public void setConcurrentRequests(AtomicLong concurrentRequests) { this.concurrentRequests = concurrentRequests; } - } diff --git a/fizz-core/src/main/java/we/stats/TimeSlot.java b/fizz-core/src/main/java/we/stats/TimeSlot.java index 433bff1..2b89508 100644 --- a/fizz-core/src/main/java/we/stats/TimeSlot.java +++ b/fizz-core/src/main/java/we/stats/TimeSlot.java @@ -91,6 +91,30 @@ public class TimeSlot { private AtomicLong gradualRejectNum = new AtomicLong(0); + private AtomicInteger _2xxStatusCount = new AtomicInteger(0); + + private AtomicInteger _4xxStatusCount = new AtomicInteger(0); + + private AtomicInteger _5xxStatusCount = new AtomicInteger(0); + + private AtomicInteger _504StatusCount = new AtomicInteger(0); + + public AtomicInteger get2xxStatusCount() { + return _2xxStatusCount; + } + + public AtomicInteger get4xxStatusCount() { + return _4xxStatusCount; + } + + public AtomicInteger get5xxStatusCount() { + return _5xxStatusCount; + } + + public AtomicInteger get504StatusCount() { + return _504StatusCount; + } + public AtomicReference getCircuitBreakState() { return circuitBreakState; } diff --git a/fizz-core/src/main/java/we/stats/TimeWindowStat.java b/fizz-core/src/main/java/we/stats/TimeWindowStat.java index 50f35b7..11efe46 100644 --- a/fizz-core/src/main/java/we/stats/TimeWindowStat.java +++ b/fizz-core/src/main/java/we/stats/TimeWindowStat.java @@ -90,6 +90,11 @@ public class TimeWindowStat { */ private Long totalBlockRequests; + private int _2xxStatus = 0; + private int _4xxStatus = 0; + private int _5xxStatus = 0; + private int _504Status = 0; + public Long getBlockRequests() { return blockRequests; } @@ -194,4 +199,35 @@ public class TimeWindowStat { this.peakRps = peakRps; } + public void set2xxStatus(int _2xxStatus) { + this._2xxStatus = _2xxStatus; + } + + public void set4xxStatus(int _4xxStatus) { + this._4xxStatus = _4xxStatus; + } + + public void set5xxStatus(int _5xxStatus) { + this._5xxStatus = _5xxStatus; + } + + public void set504Status(int _504status) { + this._504Status = _504status; + } + + public int get2xxStatus() { + return _2xxStatus; + } + + public int get4xxStatus() { + return _4xxStatus; + } + + public int get5xxStatus() { + return _5xxStatus; + } + + public int get504Status() { + return _504Status; + } } diff --git a/fizz-core/src/main/java/we/stats/ratelimit/ResourceRateLimitConfig.java b/fizz-core/src/main/java/we/stats/ratelimit/ResourceRateLimitConfig.java index 37ab5f7..2634d0e 100644 --- a/fizz-core/src/main/java/we/stats/ratelimit/ResourceRateLimitConfig.java +++ b/fizz-core/src/main/java/we/stats/ratelimit/ResourceRateLimitConfig.java @@ -39,6 +39,7 @@ public class ResourceRateLimitConfig { static final byte APP_DEFAULT = 5; static final byte APP = 6; static final byte IP = 7; + static final byte HOST = 8; } public boolean isDeleted = false; diff --git a/fizz-core/src/main/java/we/util/WebUtils.java b/fizz-core/src/main/java/we/util/WebUtils.java index 92885cb..4f27ffe 100644 --- a/fizz-core/src/main/java/we/util/WebUtils.java +++ b/fizz-core/src/main/java/we/util/WebUtils.java @@ -115,6 +115,8 @@ public abstract class WebUtils { public static final String BODY_ENCRYPT = "b-ecyt"; + public static final String ORIGINAL_ERROR = "origerr@"; + private WebUtils() { } diff --git a/fizz-core/src/test/java/we/stats/FlowStatTests.java b/fizz-core/src/test/java/we/stats/FlowStatTests.java index 97b339e..4cd07ee 100644 --- a/fizz-core/src/test/java/we/stats/FlowStatTests.java +++ b/fizz-core/src/test/java/we/stats/FlowStatTests.java @@ -29,6 +29,7 @@ import java.util.concurrent.TimeUnit; import org.junit.jupiter.api.Test; +import org.springframework.http.HttpStatus; import we.util.JacksonUtils; /** @@ -178,7 +179,7 @@ public class FlowStatTests { assertEquals("testIncrRequestResultByResourceChain_service1", result.getBlockedResourceId()); assertEquals(BlockType.CONCURRENT_REQUEST, result.getBlockType()); - stat.addRequestRT(c1.resourceConfigs, startTimeSlotId, 1, true); + stat.addRequestRT(c1.resourceConfigs, startTimeSlotId, 1, true, HttpStatus.OK); result = stat.incrRequest(c1.resourceConfigs, startTimeSlotId); assertTrue(result.isSuccess()); @@ -197,7 +198,7 @@ public class FlowStatTests { assertEquals("testIncrRequestResultByResourceChain_service2", result.getBlockedResourceId()); assertEquals(BlockType.QPS, result.getBlockType()); - stat.addRequestRT(c2.resourceConfigs, startTimeSlotId, 1, true); + stat.addRequestRT(c2.resourceConfigs, startTimeSlotId, 1, true, HttpStatus.OK); result = stat.incrRequest(c2.resourceConfigs, startTimeSlotId); assertTrue(!result.isSuccess()); diff --git a/fizz-plugin/pom.xml b/fizz-plugin/pom.xml index c6c0660..5e2ec3e 100644 --- a/fizz-plugin/pom.xml +++ b/fizz-plugin/pom.xml @@ -5,7 +5,7 @@ fizz-gateway-community com.fizzgate - 2.5.2 + 2.6 ../pom.xml 4.0.0 diff --git a/fizz-spring-boot-starter/pom.xml b/fizz-spring-boot-starter/pom.xml index 24aaee6..8ce2fdc 100644 --- a/fizz-spring-boot-starter/pom.xml +++ b/fizz-spring-boot-starter/pom.xml @@ -5,7 +5,7 @@ fizz-gateway-community com.fizzgate - 2.5.2 + 2.6 ../pom.xml 4.0.0 diff --git a/fizz-spring-boot-starter/src/main/resources/META-INF/spring.factories b/fizz-spring-boot-starter/src/main/resources/META-INF/spring.factories index 2a375f2..184e664 100644 --- a/fizz-spring-boot-starter/src/main/resources/META-INF/spring.factories +++ b/fizz-spring-boot-starter/src/main/resources/META-INF/spring.factories @@ -47,3 +47,6 @@ we.proxy.RpcInstanceServiceImpl,\ we.stats.ratelimit.ResourceRateLimitConfigService,\ we.global_resource.GlobalResourceService,\ we.dedicated_line.DedicatedLineWebServer + +# Application Listeners +org.springframework.context.ApplicationListener=we.context.event.FizzApplicationListener diff --git a/pom.xml b/pom.xml index 549194c..5fec0b2 100644 --- a/pom.xml +++ b/pom.xml @@ -12,7 +12,7 @@ 2.2.6.RELEASE 4.1.74.Final 4.4.15 - 2.17.1 + 2.17.2 1.7.36 2.7.5 1.16.1 @@ -22,7 +22,7 @@ 0.8.2 0.9.11 2.11.1 - 2.0.48.Final + 2.0.50.Final 2.2.9.RELEASE 1.7.1 1.30 @@ -38,7 +38,7 @@ fizz-gateway-community ${project.artifactId} fizz gateway community - 2.5.2 + 2.6 pom fizz-common @@ -443,7 +443,7 @@ cn.hutool hutool-crypto - 5.7.21 + 5.7.22