fizzgate/fizz-gateway-node
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

performance tuning again

Browse Source
This commit is contained in:
hongqiaowei
2021-05-28 09:42:34 +08:00
parent 22628040d5
commit 36e8866bbc
18 changed files with 396 additions and 262 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
fizz-bootstrap/pom.xml
View File

@@ -19,7 +19,7 @@
<spring-framework.version>5.2.15.RELEASE</spring-framework.version>
<reactor-bom.version>Dysprosium-SR20</reactor-bom.version>
<lettuce.version>5.3.7.RELEASE</lettuce.version>
<netty.version>4.1.63.Final</netty.version>
<netty.version>4.1.65.Final</netty.version>
<httpcore.version>4.4.14</httpcore.version>
<log4j2.version>2.13.3</log4j2.version>
<commons-lang3.version>3.12.0</commons-lang3.version>

1
fizz-bootstrap/sh/boot.sh
View File

@@ -71,6 +71,7 @@ DEFAULT_JAVA_OPTS="-XX:+AggressiveOpts \
-Xloggc:${APP_LOG_DIR}/${START_DATE_TIME}.gc \
-XX:+HeapDumpOnOutOfMemoryError \
-XX:HeapDumpPath=${APP_LOG_DIR}/dump.logs \
-Dreactor.netty.pool.maxIdleTime=120000 \
-Dorg.jboss.netty.epollBugWorkaround=true "
MEM_OPTS=${JAVA_MEM_OPTS:-$DEFAULT_JAVA_MEM_OPTS}

142
fizz-bootstrap/src/main/java/we/FizzBootstrapApplication.java
View File

@@ -21,8 +21,65 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.autoconfigure.admin.SpringApplicationAdminJmxAutoConfiguration;
import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration;
import org.springframework.boot.autoconfigure.batch.BatchAutoConfiguration;
import org.springframework.boot.autoconfigure.cache.CacheAutoConfiguration;
import org.springframework.boot.autoconfigure.cassandra.CassandraAutoConfiguration;
import org.springframework.boot.autoconfigure.couchbase.CouchbaseAutoConfiguration;
import org.springframework.boot.autoconfigure.data.cassandra.CassandraReactiveDataAutoConfiguration;
import org.springframework.boot.autoconfigure.data.couchbase.CouchbaseReactiveDataAutoConfiguration;
import org.springframework.boot.autoconfigure.data.elasticsearch.ElasticsearchDataAutoConfiguration;
import org.springframework.boot.autoconfigure.data.elasticsearch.ElasticsearchRepositoriesAutoConfiguration;
import org.springframework.boot.autoconfigure.data.jpa.JpaRepositoriesAutoConfiguration;
import org.springframework.boot.autoconfigure.data.ldap.LdapRepositoriesAutoConfiguration;
import org.springframework.boot.autoconfigure.data.mongo.MongoDataAutoConfiguration;
import org.springframework.boot.autoconfigure.data.mongo.MongoReactiveDataAutoConfiguration;
import org.springframework.boot.autoconfigure.data.mongo.MongoRepositoriesAutoConfiguration;
import org.springframework.boot.autoconfigure.data.neo4j.Neo4jDataAutoConfiguration;
import org.springframework.boot.autoconfigure.data.redis.RedisAutoConfiguration;
import org.springframework.boot.autoconfigure.data.redis.RedisReactiveAutoConfiguration;
import org.springframework.boot.autoconfigure.data.solr.SolrRepositoriesAutoConfiguration;
import org.springframework.boot.autoconfigure.data.web.SpringDataWebAutoConfiguration;
import org.springframework.boot.autoconfigure.flyway.FlywayAutoConfiguration;
import org.springframework.boot.autoconfigure.freemarker.FreeMarkerAutoConfiguration;
import org.springframework.boot.autoconfigure.h2.H2ConsoleAutoConfiguration;
import org.springframework.boot.autoconfigure.hateoas.HypermediaAutoConfiguration;
import org.springframework.boot.autoconfigure.hazelcast.HazelcastAutoConfiguration;
import org.springframework.boot.autoconfigure.hazelcast.HazelcastJpaDependencyAutoConfiguration;
import org.springframework.boot.autoconfigure.influx.InfluxDbAutoConfiguration;
import org.springframework.boot.autoconfigure.integration.IntegrationAutoConfiguration;
import org.springframework.boot.autoconfigure.jdbc.*;
import org.springframework.boot.autoconfigure.jms.JmsAutoConfiguration;
import org.springframework.boot.autoconfigure.jms.JndiConnectionFactoryAutoConfiguration;
import org.springframework.boot.autoconfigure.jms.activemq.ActiveMQAutoConfiguration;
import org.springframework.boot.autoconfigure.jms.artemis.ArtemisAutoConfiguration;
import org.springframework.boot.autoconfigure.jmx.JmxAutoConfiguration;
import org.springframework.boot.autoconfigure.jooq.JooqAutoConfiguration;
import org.springframework.boot.autoconfigure.kafka.KafkaAutoConfiguration;
import org.springframework.boot.autoconfigure.ldap.LdapAutoConfiguration;
import org.springframework.boot.autoconfigure.ldap.embedded.EmbeddedLdapAutoConfiguration;
import org.springframework.boot.autoconfigure.liquibase.LiquibaseAutoConfiguration;
import org.springframework.boot.autoconfigure.mail.MailSenderAutoConfiguration;
import org.springframework.boot.autoconfigure.mail.MailSenderValidatorAutoConfiguration;
import org.springframework.boot.autoconfigure.mongo.MongoAutoConfiguration;
import org.springframework.boot.autoconfigure.mongo.MongoReactiveAutoConfiguration;
import org.springframework.boot.autoconfigure.mongo.embedded.EmbeddedMongoAutoConfiguration;
import org.springframework.boot.autoconfigure.mustache.MustacheAutoConfiguration;
import org.springframework.boot.autoconfigure.orm.jpa.HibernateJpaAutoConfiguration;
import org.springframework.boot.autoconfigure.quartz.QuartzAutoConfiguration;
import org.springframework.boot.autoconfigure.rsocket.RSocketMessagingAutoConfiguration;
import org.springframework.boot.autoconfigure.rsocket.RSocketRequesterAutoConfiguration;
import org.springframework.boot.autoconfigure.rsocket.RSocketServerAutoConfiguration;
import org.springframework.boot.autoconfigure.rsocket.RSocketStrategiesAutoConfiguration;
import org.springframework.boot.autoconfigure.security.oauth2.client.reactive.ReactiveOAuth2ClientAutoConfiguration;
import org.springframework.boot.autoconfigure.security.oauth2.client.servlet.OAuth2ClientAutoConfiguration;
import org.springframework.boot.autoconfigure.security.oauth2.resource.reactive.ReactiveOAuth2ResourceServerAutoConfiguration;
import org.springframework.boot.autoconfigure.security.rsocket.RSocketSecurityAutoConfiguration;
import org.springframework.boot.autoconfigure.solr.SolrAutoConfiguration;
import org.springframework.boot.autoconfigure.thymeleaf.ThymeleafAutoConfiguration;
import org.springframework.boot.autoconfigure.transaction.TransactionAutoConfiguration;
import org.springframework.boot.autoconfigure.transaction.jta.JtaAutoConfiguration;
import org.springframework.boot.autoconfigure.web.reactive.error.ErrorWebFluxAutoConfiguration;
import org.springframework.boot.web.reactive.context.AnnotationConfigReactiveWebServerApplicationContext;
import org.springframework.cloud.client.discovery.EnableDiscoveryClient;
@@ -38,7 +95,90 @@ import we.log.LogSendAppender;
* @author zhongjie
*/
@SpringBootApplication(
exclude = {ErrorWebFluxAutoConfiguration.class, RedisAutoConfiguration.class, RedisReactiveAutoConfiguration.class}
exclude = {
ErrorWebFluxAutoConfiguration.class,
RedisAutoConfiguration.class,
RedisReactiveAutoConfiguration.class,
EmbeddedLdapAutoConfiguration.class,
LdapAutoConfiguration.class,
LdapRepositoriesAutoConfiguration.class,
JndiConnectionFactoryAutoConfiguration.class,
JndiDataSourceAutoConfiguration.class,
HypermediaAutoConfiguration.class,
MustacheAutoConfiguration.class,
ThymeleafAutoConfiguration.class,
FreeMarkerAutoConfiguration.class,
RSocketMessagingAutoConfiguration.class,
RSocketRequesterAutoConfiguration.class,
RSocketSecurityAutoConfiguration.class,
RSocketServerAutoConfiguration.class,
RSocketStrategiesAutoConfiguration.class,
SpringDataWebAutoConfiguration.class,
DataSourceAutoConfiguration.class,
DataSourceTransactionManagerAutoConfiguration.class,
XADataSourceAutoConfiguration.class,
H2ConsoleAutoConfiguration.class,
JdbcTemplateAutoConfiguration.class,
JtaAutoConfiguration.class,
TransactionAutoConfiguration.class,
FlywayAutoConfiguration.class,
InfluxDbAutoConfiguration.class,
LiquibaseAutoConfiguration.class,
JpaRepositoriesAutoConfiguration.class,
HibernateJpaAutoConfiguration.class,
JooqAutoConfiguration.class,
MongoAutoConfiguration.class,
EmbeddedMongoAutoConfiguration.class,
MongoReactiveAutoConfiguration.class,
MongoDataAutoConfiguration.class,
MongoRepositoriesAutoConfiguration.class,
MongoReactiveDataAutoConfiguration.class,
CouchbaseAutoConfiguration.class,
CouchbaseReactiveDataAutoConfiguration.class,
CassandraAutoConfiguration.class,
CassandraReactiveDataAutoConfiguration.class,
SolrAutoConfiguration.class,
SolrRepositoriesAutoConfiguration.class,
ElasticsearchDataAutoConfiguration.class,
ElasticsearchRepositoriesAutoConfiguration.class,
JmsAutoConfiguration.class,
ActiveMQAutoConfiguration.class,
KafkaAutoConfiguration.class,
ArtemisAutoConfiguration.class,
RabbitAutoConfiguration.class,
MailSenderAutoConfiguration.class,
MailSenderValidatorAutoConfiguration.class,
Neo4jDataAutoConfiguration.class,
HazelcastAutoConfiguration.class,
HazelcastJpaDependencyAutoConfiguration.class,
CacheAutoConfiguration.class,
BatchAutoConfiguration.class,
IntegrationAutoConfiguration.class,
JmxAutoConfiguration.class,
SpringApplicationAdminJmxAutoConfiguration.class,
OAuth2ClientAutoConfiguration.class,
ReactiveOAuth2ClientAutoConfiguration.class,
ReactiveOAuth2ResourceServerAutoConfiguration.class,
QuartzAutoConfiguration.class
}
)
@NacosPropertySource(dataId = "${nacos.config.data-id}", groupId = "${nacos.config.group}", autoRefreshed = true)
@EnableDiscoveryClient

12
fizz-bootstrap/src/main/resources/log4j2-spring.xml
View File

@@ -1,9 +1,9 @@
<?xml version="1.0" encoding="UTF-8"?>
<Configuration status="info" packages="we.log">
<properties>
<property name="APP_NAME">${sys:APP_NAME}</property>
</properties>
<Configuration status="warn">
<properties>
<property name="APP_NAME">${sys:APP_NAME}</property>
</properties>
<Appenders>
<Console name="Console" target="SYSTEM_OUT">
<PatternLayout pattern="%d{yyyy-MM-dd HH:mm:ss.SSS} [%t] %level %logger{36} - %X{traceId} %msg%n" />
@@ -13,10 +13,10 @@
</LogSend>
</Appenders>
<Loggers>
<Root level="info">
<Root level="warn">
<AppenderRef ref="Console" />
<AppenderRef ref="LogSend" />
</Root>
<Logger name="we" level="DEBUG"/>
<Logger name="we" level="warn" additivity="false" />
</Loggers>
</Configuration>

162
fizz-common/src/main/java/we/config/WebClientConfig.java
View File

@@ -23,15 +23,13 @@ import io.netty.handler.timeout.WriteTimeoutHandler;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.http.client.reactive.ReactorClientHttpConnector;
import org.springframework.http.client.reactive.ReactorResourceFactory;
import org.springframework.web.reactive.function.client.ExchangeStrategies;
import org.springframework.web.reactive.function.client.WebClient;
import reactor.netty.http.client.HttpClient;
import reactor.netty.resources.ConnectionProvider;
import reactor.netty.resources.LoopResources;
import we.util.JacksonUtils;
import reactor.netty.tcp.TcpClient;
import javax.annotation.Resource;
import java.time.Duration;
import java.util.concurrent.TimeUnit;
/**
@@ -42,144 +40,122 @@ public abstract class WebClientConfig {
protected static final Logger log = LoggerFactory.getLogger(WebClientConfig.class);
// private String name;
//
// private int maxConnections = 2_000;
//
// private Duration maxIdleTime = Duration.ofMillis(40_000);
//
// private Duration pendingAcquireTimeout = Duration.ofMillis(6_000);
private Long connReadTimeout = null; // 20_000
private long connReadTimeout = 20_000;
private Long connWriteTimeout = null; // 20_000
private long connWriteTimeout = 20_000;
private Integer chConnTimeout = null; // 20_000;
private int chConnTimeout = 20_000;
private Long responseTimeout = null; // 20_000
private boolean chTcpNodelay = true;
private Boolean chTcpNodelay = null; // true
private boolean chSoKeepAlive = true;
private Boolean chSoKeepAlive = null; // true
private boolean compress = true;
private Boolean compress = null; // true
// public String getName() {
// return name;
// }
//
// public void setName(String name) {
// this.name = name;
// }
//
// public int getMaxConnections() {
// return maxConnections;
// }
//
// public void setMaxConnections(int maxConnections) {
// this.maxConnections = maxConnections;
// }
//
// public Duration getMaxIdleTime() {
// return maxIdleTime;
// }
//
// public void setMaxIdleTime(long maxIdleTime) {
// this.maxIdleTime = Duration.ofMillis(maxIdleTime);
// }
//
// public Duration getPendingAcquireTimeout() {
// return pendingAcquireTimeout;
// }
//
// public void setPendingAcquireTimeout(long pendingAcquireTimeout) {
// this.pendingAcquireTimeout = Duration.ofMillis(pendingAcquireTimeout);
// }
public long getConnReadTimeout() {
public Long getConnReadTimeout() {
return connReadTimeout;
}
public void setConnReadTimeout(long connReadTimeout) {
public void setConnReadTimeout(Long connReadTimeout) {
this.connReadTimeout = connReadTimeout;
}
public long getConnWriteTimeout() {
public Long getConnWriteTimeout() {
return connWriteTimeout;
}
public void setConnWriteTimeout(long connWriteTimeout) {
public void setConnWriteTimeout(Long connWriteTimeout) {
this.connWriteTimeout = connWriteTimeout;
}
public int getChConnTimeout() {
public Integer getChConnTimeout() {
return chConnTimeout;
}
public void setChConnTimeout(int chConnTimeout) {
public void setChConnTimeout(Integer chConnTimeout) {
this.chConnTimeout = chConnTimeout;
}
public boolean isChTcpNodelay() {
public Long getResponseTimeout() {
return responseTimeout;
}
public void setResponseTimeout(Long responseTimeout) {
this.responseTimeout = responseTimeout;
}
public Boolean isChTcpNodelay() {
return chTcpNodelay;
}
public void setChTcpNodelay(boolean chTcpNodelay) {
public void setChTcpNodelay(Boolean chTcpNodelay) {
this.chTcpNodelay = chTcpNodelay;
}
public boolean isChSoKeepAlive() {
public Boolean isChSoKeepAlive() {
return chSoKeepAlive;
}
public void setChSoKeepAlive(boolean chSoKeepAlive) {
public void setChSoKeepAlive(Boolean chSoKeepAlive) {
this.chSoKeepAlive = chSoKeepAlive;
}
public boolean isCompress() {
public Boolean isCompress() {
return compress;
}
public void setCompress(boolean compress) {
public void setCompress(Boolean compress) {
this.compress = compress;
}
@Resource
ReactorResourceFactory reactorResourceFactory;
// @Resource
// ReactorClientHttpConnector reactorClientHttpConnector;
@Resource
WebClient.Builder webClientBuilder;
public WebClient webClient() {
ConnectionProvider cp = reactorResourceFactory.getConnectionProvider();
LoopResources lrs = reactorResourceFactory.getLoopResources();
HttpClient httpClient = HttpClient.create(cp).compress(compress).tcpConfiguration(
tcpClient -> {
return tcpClient.runOn(lrs)
// .bootstrap(
// bootstrap -> (
// bootstrap.channel(NioSocketChannel.class)
// )
// )
.doOnConnected(
connection -> {
connection.addHandlerLast(new ReadTimeoutHandler( connReadTimeout, TimeUnit.MILLISECONDS))
.addHandlerLast(new WriteTimeoutHandler(connWriteTimeout, TimeUnit.MILLISECONDS));
}
)
.option(ChannelOption.CONNECT_TIMEOUT_MILLIS, chConnTimeout)
.option(ChannelOption.TCP_NODELAY, chTcpNodelay)
.option(ChannelOption.SO_KEEPALIVE, chSoKeepAlive)
// .option(ChannelOption.ALLOCATOR, PreferHeapByteBufAllocator.DEFAULT)
// .option(ChannelOption.ALLOCATOR, PooledByteBufAllocator.DEFAULT)
// .option(ChannelOption.ALLOCATOR, UnpooledByteBufAllocator.DEFAULT)
;
}
);
return WebClient.builder().exchangeStrategies(
HttpClient httpClient = HttpClient.create()
.tcpConfiguration(
tcpClient -> {
TcpClient newTcpClient = tcpClient.doOnConnected(
connection -> {
if (connReadTimeout != null) {
connection.addHandlerLast(new ReadTimeoutHandler(connReadTimeout, TimeUnit.MILLISECONDS));
}
if (connWriteTimeout != null) {
connection.addHandlerLast(new WriteTimeoutHandler(connWriteTimeout, TimeUnit.MILLISECONDS));
}
}
);
if (chConnTimeout != null) {
newTcpClient = newTcpClient.option(ChannelOption.CONNECT_TIMEOUT_MILLIS, chConnTimeout);
}
if (chTcpNodelay != null) {
newTcpClient = newTcpClient.option(ChannelOption.TCP_NODELAY, chTcpNodelay);
}
if (chSoKeepAlive != null) {
newTcpClient = newTcpClient.option(ChannelOption.SO_KEEPALIVE, chSoKeepAlive);
}
return newTcpClient;
}
);
if (compress != null) {
httpClient = httpClient.compress(compress);
}
if (responseTimeout != null) {
httpClient = httpClient.responseTimeout(Duration.ofMillis(responseTimeout));
}
return webClientBuilder.exchangeStrategies(
ExchangeStrategies.builder().codecs(configurer -> configurer.defaultCodecs().maxInMemorySize(-1))
.build()
)
.clientConnector(new ReactorClientHttpConnector(httpClient))
.build();
)
.clientConnector(new ReactorClientHttpConnector(httpClient))
.build();
}
@Override

88
fizz-core/src/main/java/we/config/AggrWebClientConfig.java
View File

@@ -1,44 +1,44 @@
/*
* Copyright (C) 2020 the original author or authors.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package we.config;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.web.reactive.function.client.WebClient;
import reactor.netty.resources.LoopResources;
import we.util.JacksonUtils;
/**
* @author hongqiaowei
*/
@Configuration
@ConfigurationProperties(prefix = AggrWebClientConfig.prefix)
public class AggrWebClientConfig extends WebClientConfig {
protected static final String prefix = "aggr-webclient";
public static final String aggrWebClient = "aggrWebClient";
@Bean(aggrWebClient)
public WebClient webClient() {
log.info(aggrWebClient + ": " + this);
return super.webClient();
}
}
// /*
// * Copyright (C) 2020 the original author or authors.
// *
// * This program is free software: you can redistribute it and/or modify
// * it under the terms of the GNU General Public License as published by
// * the Free Software Foundation, either version 3 of the License, or
// * any later version.
// *
// * This program is distributed in the hope that it will be useful,
// * but WITHOUT ANY WARRANTY; without even the implied warranty of
// * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// * GNU General Public License for more details.
// *
// * You should have received a copy of the GNU General Public License
// * along with this program. If not, see <https://www.gnu.org/licenses/>.
// */
//
// package we.config;
//
// import org.springframework.boot.context.properties.ConfigurationProperties;
// import org.springframework.context.annotation.Bean;
// import org.springframework.context.annotation.Configuration;
// import org.springframework.web.reactive.function.client.WebClient;
// import reactor.netty.resources.LoopResources;
// import we.util.JacksonUtils;
//
// /**
// * @author hongqiaowei
// */
//
// @Configuration
// @ConfigurationProperties(prefix = AggrWebClientConfig.prefix)
// public class AggrWebClientConfig extends WebClientConfig {
//
// protected static final String prefix = "aggr-webclient";
//
// public static final String aggrWebClient = "aggrWebClient";
//
// @Bean(aggrWebClient)
// public WebClient webClient() {
// log.info(aggrWebClient + ": " + this);
// return super.webClient();
// }
// }

70
fizz-core/src/main/java/we/config/WebFluxConfig.java
View File

@@ -22,6 +22,7 @@ import io.netty.channel.ChannelOption;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.condition.ConditionalOnBean;
import org.springframework.boot.autoconfigure.web.ServerProperties;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
@@ -34,6 +35,8 @@ import org.springframework.web.reactive.config.EnableWebFlux;
import org.springframework.web.reactive.config.ResourceHandlerRegistry;
import org.springframework.web.reactive.config.WebFluxConfigurer;
import org.springframework.web.reactive.resource.HttpResource;
import reactor.netty.http.HttpResources;
import reactor.netty.resources.ConnectionProvider;
import reactor.netty.resources.LoopResources;
@@ -50,39 +53,48 @@ public class WebFluxConfig {
private static final Logger log = LoggerFactory.getLogger(WebFluxConfig.class);
@NacosValue(value = "${server.connection-pool.max-connections:500}", autoRefreshed = true)
@Value( "${server.connection-pool.max-connections:500}" )
private int maxConnections;
// @NacosValue(value = "${server.connection-pool.max-connections:500}", autoRefreshed = true)
// @Value( "${server.connection-pool.max-connections:500}" )
// private int maxConnections;
//
// @NacosValue(value = "${server.connection-pool.max-idle-time:30000}", autoRefreshed = true)
// @Value( "${server.connection-pool.max-idle-time:30000}" )
// private long maxIdleTime;
@NacosValue(value = "${server.connection-pool.max-idle-time:30000}", autoRefreshed = true)
@Value( "${server.connection-pool.max-idle-time:30000}" )
private long maxIdleTime;
// private ConnectionProvider connectionProvider() {
// ConnectionProvider connectionProvider = ConnectionProvider.builder("fizz-cp")
// .maxConnections(maxConnections)
// .maxIdleTime(Duration.ofMillis(maxIdleTime))
// .pendingAcquireMaxCount(-1)
// .build();
// log.info("server connection provider: maxConnections={}, maxIdleTime={}", maxConnections, maxIdleTime);
// return connectionProvider;
// }
private ConnectionProvider connectionProvider() {
ConnectionProvider connectionProvider = ConnectionProvider.builder("fizz-cp")
.maxConnections(maxConnections)
.maxIdleTime(Duration.ofMillis(maxIdleTime))
.pendingAcquireMaxCount(-1)
.build();
log.info("server connection provider: maxConnections={}, maxIdleTime={}", maxConnections, maxIdleTime);
return connectionProvider;
}
// @ConditionalOnBean(ReactorResourceFactory.class)
// @Bean(name = "$dummyObject")
// public Object dummyObject() {
// ConnectionProvider provider = connectionProvider();
// HttpResources.set(provider);
// log.info("replace default connection provider with: " + provider);
// return new Object();
// }
private LoopResources loopResources() {
LoopResources loopResources = LoopResources.create("fizz-lrs");
log.info("server loop resources: " + loopResources);
return loopResources;
}
// private LoopResources loopResources() {
// LoopResources loopResources = LoopResources.create("fizz-lrs");
// log.info("server loop resources: " + loopResources);
// return loopResources;
// }
@Bean
public ReactorResourceFactory reactorServerResourceFactory() {
ReactorResourceFactory rrf = new ReactorResourceFactory();
rrf.setUseGlobalResources(false);
rrf.setLoopResources(loopResources());
rrf.setConnectionProvider(connectionProvider());
log.info("server reactor resource factory: " + rrf);
return rrf;
}
// @Bean
// public ReactorResourceFactory reactorServerResourceFactory() {
// ReactorResourceFactory rrf = new ReactorResourceFactory();
// rrf.setUseGlobalResources(false);
// rrf.setLoopResources(loopResources());
// rrf.setConnectionProvider(connectionProvider());
// log.info("server reactor resource factory: " + rrf);
// return rrf;
// }
// @Bean
// public NettyReactiveWebServerFactory nettyReactiveWebServerFactory(ServerProperties serverProperties/*, ReactorResourceFactory reactorResourceFactory*/) {

26
fizz-core/src/main/java/we/filter/FizzLogFilter.java
View File

@@ -20,12 +20,14 @@ package we.filter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.core.annotation.Order;
import org.springframework.http.HttpStatus;
import org.springframework.stereotype.Component;
import org.springframework.web.server.ServerWebExchange;
import org.springframework.web.server.WebFilter;
import org.springframework.web.server.WebFilterChain;
import reactor.core.publisher.Mono;
import we.flume.clients.log4j2appender.LogService;
import we.util.Constants;
import we.util.ThreadContext;
import we.util.WebUtils;
@@ -37,15 +39,33 @@ import we.util.WebUtils;
@Order(0)
public class FizzLogFilter implements WebFilter {
private static final Logger LOGGER = LoggerFactory.getLogger(FizzLogFilter.class);
private static final Logger LOGGER = LoggerFactory.getLogger(FizzLogFilter.class);
private static final String resp = "\nresponse ";
private static final String resp = "\nresponse ";
private static final String in = " in ";
private static final String in = " in ";
private static final String admin = "admin";
private static final String actuator = "actuator";
public static final String ADMIN_REQUEST = "$a";
@Override
public Mono<Void> filter(ServerWebExchange exchange, WebFilterChain chain) {
long startTime = System.currentTimeMillis();
String path = exchange.getRequest().getPath().value();
int secFS = path.indexOf(Constants.Symbol.FORWARD_SLASH, 1);
if (secFS == -1) {
return WebUtils.responseError(exchange, HttpStatus.INTERNAL_SERVER_ERROR.value(), "request path should like /optional-prefix/service-name/real-biz-path");
}
String s = path.substring(1, secFS);
if (s.equals(admin) || s.equals(actuator)) {
exchange.getAttributes().put(ADMIN_REQUEST, Constants.Symbol.EMPTY);
}
return chain.filter(exchange).doAfterTerminate(
() -> {
if (LOGGER.isInfoEnabled()) {

13
fizz-core/src/main/java/we/filter/FizzWebFilter.java
View File

@@ -37,17 +37,10 @@ public abstract class FizzWebFilter implements WebFilter {
@Override
public Mono<Void> filter(ServerWebExchange exchange, WebFilterChain chain) {
String path = exchange.getRequest().getPath().value();
int secFS = path.indexOf(Constants.Symbol.FORWARD_SLASH, 1);
if (secFS == -1) {
return WebUtils.responseError(exchange, HttpStatus.INTERNAL_SERVER_ERROR.value(), "request path should like /optional-prefix/service-name/real-biz-path");
}
String s = path.substring(1, secFS);
if (s.equals(admin) || s.equals(actuator)) {
return chain.filter(exchange);
} else {
if (exchange.getAttribute(FizzLogFilter.ADMIN_REQUEST) == null) {
return doFilter(exchange, chain);
} else {
return chain.filter(exchange);
}
}

37
fizz-core/src/main/java/we/filter/PreprocessFilter.java
View File

@@ -81,12 +81,12 @@ public class PreprocessFilter extends FizzWebFilter {
.flatMap(
v -> {
Object authRes = WebUtils.getFilterResultDataItem(exchange, AuthPluginFilter.AUTH_PLUGIN_FILTER, AuthPluginFilter.RESULT);
Mono m;
Mono m = ReactorUtils.getInitiateMono();
if (authRes instanceof ApiConfig) {
ApiConfig ac = (ApiConfig) authRes;
afterAuth(exchange, ac);
m = executeFixedPluginFilters(exchange);
m = m.defaultIfEmpty(ReactorUtils.NULL);
// m = executeFixedPluginFilters(exchange);
// m = m.defaultIfEmpty(ReactorUtils.NULL);
if (ac.pluginConfigs == null || ac.pluginConfigs.isEmpty()) {
return m.flatMap(func(exchange, chain));
} else {
@@ -95,8 +95,9 @@ public class PreprocessFilter extends FizzWebFilter {
}
} else if (authRes == ApiConfigService.Access.YES) {
afterAuth(exchange, null);
m = executeFixedPluginFilters(exchange);
return m.defaultIfEmpty(ReactorUtils.NULL).flatMap(func(exchange, chain));
// m = executeFixedPluginFilters(exchange);
// return m.defaultIfEmpty(ReactorUtils.NULL).flatMap(func(exchange, chain));
return m.flatMap(func(exchange, chain));
} else {
ApiConfigService.Access access = (ApiConfigService.Access) authRes;
return WebUtils.responseError(exchange, HttpStatus.FORBIDDEN.value(), access.getReason());
@@ -142,19 +143,19 @@ public class PreprocessFilter extends FizzWebFilter {
};
}
private Mono<Void> executeFixedPluginFilters(ServerWebExchange exchange) {
Mono vm = Mono.empty();
List<FixedPluginFilter> fixedPluginFilters = FixedPluginFilter.getPluginFilters();
for (byte i = 0; i < fixedPluginFilters.size(); i++) {
FixedPluginFilter fpf = fixedPluginFilters.get(i);
vm = vm.defaultIfEmpty(ReactorUtils.NULL).flatMap(
v -> {
return fpf.filter(exchange, null, null);
}
);
}
return vm;
}
// private Mono<Void> executeFixedPluginFilters(ServerWebExchange exchange) {
// Mono vm = Mono.empty();
// List<FixedPluginFilter> fixedPluginFilters = FixedPluginFilter.getPluginFilters();
// for (byte i = 0; i < fixedPluginFilters.size(); i++) {
// FixedPluginFilter fpf = fixedPluginFilters.get(i);
// vm = vm.defaultIfEmpty(ReactorUtils.NULL).flatMap(
// v -> {
// return fpf.filter(exchange, null, null);
// }
// );
// }
// return vm;
// }
private Mono<Void> executeManagedPluginFilters(ServerWebExchange exchange, List<PluginConfig> pluginConfigs) {
Mono vm = Mono.empty();

6
fizz-core/src/main/java/we/filter/RouteFilter.java
View File

@@ -28,6 +28,7 @@ import org.springframework.http.server.reactive.ServerHttpResponse;
import org.springframework.stereotype.Component;
import org.springframework.web.reactive.function.BodyExtractors;
import org.springframework.web.reactive.function.client.ClientResponse;
import org.springframework.web.reactive.function.client.WebClient;
import org.springframework.web.server.ServerWebExchange;
import org.springframework.web.server.WebFilterChain;
import reactor.core.publisher.Mono;
@@ -59,6 +60,7 @@ public class RouteFilter extends FizzWebFilter {
@Override
public Mono<Void> doFilter(ServerWebExchange exchange, WebFilterChain chain) {
FilterResult pfr = WebUtils.getPrevFilterResult(exchange);
if (pfr.success) {
return doFilter0(exchange, chain);
@@ -96,7 +98,9 @@ public class RouteFilter extends FizzWebFilter {
return send(exchange, WebUtils.getBackendService(exchange), pathQuery, hdrs);
} else if (ac.type == ApiConfig.Type.REVERSE_PROXY) {
String uri = ac.getNextHttpHostPort() + WebUtils.appendQuery(WebUtils.getBackendPath(exchange), exchange);
String uri = ThreadContext.getStringBuilder().append(ac.getNextHttpHostPort())
.append(WebUtils.appendQuery(WebUtils.getBackendPath(exchange), exchange))
.toString();
return fizzWebClient.send(rid, clientReq.getMethod(), uri, hdrs, clientReq.getBody()).flatMap(genServerResponse(exchange));
} else {

13
fizz-core/src/main/java/we/plugin/auth/ApiConfig.java
View File

@@ -84,7 +84,7 @@ public class ApiConfig {
@JsonProperty("proxyMode")
public byte type = Type.SERVICE_DISCOVERY;
private AtomicInteger counter = new AtomicInteger(-1);
private int counter = 0;
public List<String> httpHostPorts;
@@ -161,12 +161,13 @@ public class ApiConfig {
@JsonIgnore
public String getNextHttpHostPort() {
int idx = counter.incrementAndGet();
if (idx < 0) {
counter.set(0);
idx = 0;
int i = counter++;
if (i < 0) {
i = Math.abs(i);
}
return httpHostPorts.get(idx % httpHostPorts.size());
return httpHostPorts.get(
i % httpHostPorts.size()
);
}
public String transform(String reqPath) {

6
fizz-core/src/main/java/we/plugin/auth/ApiConfigService.java
View File

@@ -263,7 +263,8 @@ public class ApiConfigService {
List<ApiConfig> apiConfigs = sc.getApiConfigs(method, path, gatewayGroup);
if (!apiConfigs.isEmpty()) {
List<String> matchPathPatterns = ThreadContext.getArrayList(mpps, String.class);
for (ApiConfig ac : apiConfigs) {
for (int i = 0; i < apiConfigs.size(); i++) {
ApiConfig ac = apiConfigs.get(i);
if (ac.checkApp) {
if (apiConifg2appsService.contains(ac.id, app)) {
matchPathPatterns.add(ac.path);
@@ -279,7 +280,8 @@ public class ApiConfigService {
Collections.sort(matchPathPatterns, UrlTransformUtils.ANT_PATH_MATCHER.getPatternComparator(path));
}
String bestPathPattern = matchPathPatterns.get(0);
for (ApiConfig ac : apiConfigs) {
for (int i = 0; i < apiConfigs.size(); i++) {
ApiConfig ac = apiConfigs.get(i);
if (StringUtils.equals(ac.path, bestPathPattern)) {
return ac;
}

3
fizz-core/src/main/java/we/plugin/auth/ServiceConfig.java
View File

@@ -142,7 +142,8 @@ public class ServiceConfig {
return Collections.emptyList();
} else {
List<ApiConfig> lst = ThreadContext.getArrayList(acs, ApiConfig.class);
for (GatewayGroup2apiConfig gatewayGroup2apiConfig : matchGatewayGroup2apiConfigs) {
for (int i = 0; i < matchGatewayGroup2apiConfigs.size(); i++) {
GatewayGroup2apiConfig gatewayGroup2apiConfig = matchGatewayGroup2apiConfigs.get(i);
Set<ApiConfig> apiConfigs = gatewayGroup2apiConfig.get(gatewayGroup);
if (apiConfigs != null) {
lst.addAll(apiConfigs);

64
fizz-core/src/main/java/we/proxy/FizzWebClient.java
View File

@@ -18,7 +18,6 @@
package we.proxy;
import com.alibaba.nacos.api.config.annotation.NacosValue;
import com.google.common.collect.Lists;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -33,8 +32,6 @@ import org.springframework.web.reactive.function.client.ClientResponse;
import org.springframework.web.reactive.function.client.WebClient;
import reactor.core.publisher.Flux;
import reactor.core.publisher.Mono;
import reactor.core.scheduler.Schedulers;
import we.config.AggrWebClientConfig;
import we.config.ProxyWebClientConfig;
import we.flume.clients.log4j2appender.LogService;
import we.util.Constants;
@@ -43,11 +40,8 @@ import we.util.WebUtils;
import javax.annotation.PostConstruct;
import javax.annotation.Resource;
import java.time.Duration;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.ThreadLocalRandom;
/**
* @author hongqiaowei
@@ -62,7 +56,7 @@ public class FizzWebClient {
private static final String localhost = "localhost";
private static final String host = "HOST";
private static final String host = "Host";
@Resource
private DiscoveryClientUriSelector discoveryClientUriSelector;
@@ -70,8 +64,8 @@ public class FizzWebClient {
@Resource(name = ProxyWebClientConfig.proxyWebClient)
private WebClient proxyWebClient;
@Resource(name = AggrWebClientConfig.aggrWebClient)
private WebClient aggrWebClient;
// @Resource(name = AggrWebClientConfig.aggrWebClient)
// private WebClient aggrWebClient;
@NacosValue(value = "${fizz-web-client.timeout:-1}")
@Value("${fizz-web-client.timeout:-1}")
@@ -88,18 +82,18 @@ public class FizzWebClient {
public Mono<ClientResponse> aggrSend(String aggrService, HttpMethod aggrMethod, String aggrPath, @Nullable String originReqIdOrBizId,
HttpMethod method, String uriOrSvc, @Nullable HttpHeaders headers, @Nullable Object body, @Nullable Long timeout) {
ThreadContext.set(aggrSend, Constants.Symbol.EMPTY); // TODO will be remove in future
// ThreadContext.set(aggrSend, Constants.Symbol.EMPTY); // TODO will be remove in future
CallBackendConfig cbc = null;
if (timeout != null) {
cbc = new CallBackendConfig(timeout);
}
// if (timeout != null) {
// cbc = new CallBackendConfig(timeout);
// }
return aggrResolveAddressSend(aggrService, aggrMethod, aggrPath, originReqIdOrBizId, method, uriOrSvc, headers, body, cbc);
}
public Mono<ClientResponse> aggrSend(String aggrService, HttpMethod aggrMethod, String aggrPath, @Nullable String originReqIdOrBizId,
HttpMethod method, String uriOrSvc, @Nullable HttpHeaders headers, @Nullable Object body) {
ThreadContext.set(aggrSend, Constants.Symbol.EMPTY); // TODO will be remove in future
// ThreadContext.set(aggrSend, Constants.Symbol.EMPTY); // TODO will be remove in future
return aggrResolveAddressSend(aggrService, aggrMethod, aggrPath, originReqIdOrBizId, method, uriOrSvc, headers, body, null);
}
@@ -152,38 +146,28 @@ public class FizzWebClient {
@Nullable HttpHeaders headers, @Nullable Object body, @Nullable Long timeout) {
CallBackendConfig cbc = null;
if (timeout != null) {
cbc = new CallBackendConfig(timeout);
}
// if (timeout != null) {
// cbc = new CallBackendConfig(timeout);
// }
return send2uri(originReqIdOrBizId, method, uri, headers, body, cbc);
}
private static final String r = "R";
private Mono<ClientResponse> send2uri(@Nullable String originReqIdOrBizId, HttpMethod method, String uri,
@Nullable HttpHeaders headers, @Nullable Object body, @Nullable CallBackendConfig cbc) {
if (originReqIdOrBizId == null) { // should not execute this
if (headers == null) {
originReqIdOrBizId = r + ThreadLocalRandom.current().nextInt(1_000, 10_000);
} else {
originReqIdOrBizId = r + headers.hashCode();
}
}
final String reqId = originReqIdOrBizId;
if (log.isDebugEnabled()) {
StringBuilder b = ThreadContext.getStringBuilder();
WebUtils.request2stringBuilder(reqId, method, uri, headers, null, b);
log.debug(b.toString(), LogService.BIZ_ID, reqId);
WebUtils.request2stringBuilder(originReqIdOrBizId, method, uri, headers, null, b);
log.debug(b.toString(), LogService.BIZ_ID, originReqIdOrBizId);
}
if (cbc == null) {
cbc = CallBackendConfig.DEFAULT;
}
// if (cbc == null) {
// cbc = CallBackendConfig.DEFAULT;
// }
// TODO remove this, and all event loop share one web client or one event loop one web client in future
WebClient.RequestBodySpec req = (ThreadContext.remove(aggrSend) == null ? proxyWebClient : aggrWebClient).method(method).uri(uri).headers(
// WebClient.RequestBodySpec req = (ThreadContext.remove(aggrSend) == null ? proxyWebClient : aggrWebClient).method(method).uri(uri).headers(
WebClient.RequestBodySpec req = proxyWebClient.method(method).uri(uri).headers(
hdrs -> {
if (headers != null) {
headers.forEach(
@@ -208,7 +192,7 @@ public class FizzWebClient {
}
}
Mono<ClientResponse> rm = req.exchange()
return req.exchange()
/*
.name(reqId)
.doOnRequest(i -> {})
@@ -224,10 +208,9 @@ public class FizzWebClient {
*/
;
if (log.isDebugEnabled()) {
rm = rm.log();
}
return rm;
// if (log.isDebugEnabled()) {
// rm = rm.log();
// }
// TODO 请求完成后，做metric, 以反哺后续的请求转发
}
@@ -265,10 +248,11 @@ public class FizzWebClient {
private boolean isService(String s) {
if (StringUtils.indexOfAny(s, Constants.Symbol.DOT, Constants.Symbol.COLON) > 0
|| StringUtils.indexOfIgnoreCase(s, localhost) > 0) {
|| StringUtils.startsWith(s, localhost)) {
return false;
} else {
return true;
}
}
}

10
fizz-core/src/main/java/we/util/WebUtils.java
View File

@@ -187,7 +187,7 @@ public abstract class WebUtils {
}
public static Map<String, FilterResult> getFilterContext(ServerWebExchange exchange) {
return (Map<String, FilterResult>) exchange.getAttributes().get(FILTER_CONTEXT);
return (Map<String, FilterResult>) exchange.getAttribute(FILTER_CONTEXT);
}
public static FilterResult getFilterResult(ServerWebExchange exchange, String filter) {
@@ -367,16 +367,16 @@ public abstract class WebUtils {
return path;
}
public static Map<String, String> getAppendHeaders(ServerWebExchange exchange) {
return (Map<String, String>) exchange.getAttributes().get(APPEND_HEADERS);
}
public static Map<String, String> appendHeader(ServerWebExchange exchange, String name, String value) {
Map<String, String> hdrs = getAppendHeaders(exchange);
hdrs.put(name, value);
return hdrs;
}
public static Map<String, String> getAppendHeaders(ServerWebExchange exchange) {
return (Map<String, String>) exchange.getAttribute(APPEND_HEADERS);
}
public static HttpHeaders mergeAppendHeaders(ServerWebExchange exchange) {
ServerHttpRequest req = exchange.getRequest();
Map<String, String> appendHeaders = getAppendHeaders(exchange);

1
fizz-spring-boot-starter/src/main/resources/META-INF/spring.factories
View File

@@ -1,6 +1,5 @@
org.springframework.boot.autoconfigure.EnableAutoConfiguration=\
we.config.AggregateRedisConfig,\
we.config.AggrWebClientConfig,\
we.config.ApolloConfig,\
we.config.AppConfigProperties,\
we.config.FlowControlConfig,\

2
pom.xml
View File

@@ -10,7 +10,7 @@
<reactor-bom.version>Dysprosium-SR20</reactor-bom.version>
<lettuce.version>5.3.7.RELEASE</lettuce.version>
<nacos.version>0.2.7</nacos.version>
<netty.version>4.1.63.Final</netty.version>
<netty.version>4.1.65.Final</netty.version>
<httpcore.version>4.4.14</httpcore.version>
<log4j2.version>2.13.3</log4j2.version>
<apache.dubbo.version>2.7.5</apache.dubbo.version>