diff --git a/pom.xml b/pom.xml index 4cec5ac..c817622 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ org.springframework.boot spring-boot-starter-parent - 2.2.10.RELEASE + 2.2.11.RELEASE we @@ -34,10 +34,13 @@ 1.8 - 2.13.3 - 4.1.53.Final - 4.5.13 + 5.2.11.RELEASE + Dysprosium-SR14 + 5.3.5.RELEASE 0.2.7 + 4.1.54.Final + 4.4.14 + 2.13.3 @@ -67,7 +70,7 @@ com.alibaba fastjson - 1.2.74 + 1.2.75 @@ -146,13 +149,19 @@ org.springframework.cloud spring-cloud-starter-netflix-eureka-client - 2.2.5.RELEASE + 2.2.6.RELEASE com.alibaba.boot nacos-config-spring-boot-starter ${nacos.version} + + + org.springframework.boot + spring-boot-starter-logging + + com.alibaba.boot diff --git a/src/main/java/we/filter/FilterExceptionHandlerConfig.java b/src/main/java/we/filter/FilterExceptionHandlerConfig.java index ac8eea1..a714090 100644 --- a/src/main/java/we/filter/FilterExceptionHandlerConfig.java +++ b/src/main/java/we/filter/FilterExceptionHandlerConfig.java @@ -55,8 +55,8 @@ public class FilterExceptionHandlerConfig { return resp.writeWith(Mono.just(resp.bufferFactory().wrap(ex.getData().toString().getBytes()))); } } - if (t instanceof RedirectException) { - RedirectException ex = (RedirectException) t; + if (t instanceof RedirectException) { + RedirectException ex = (RedirectException) t; if (ex.getRedirectUrl() != null) { ServerHttpResponse resp = exchange.getResponse(); resp.setStatusCode(HttpStatus.MOVED_PERMANENTLY); @@ -64,23 +64,22 @@ public class FilterExceptionHandlerConfig { return Mono.empty(); } } - if (t instanceof ExecuteScriptException) { - ExecuteScriptException ex = (ExecuteScriptException) t; - ServerHttpResponse resp = exchange.getResponse(); - resp.getHeaders().add(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_VALUE); - - RespEntity rs = null; - String reqId = exchange.getRequest().getId(); - if (ex.getStepContext() != null && ex.getStepContext().returnContext()) { - rs = new RespEntity(HttpStatus.INTERNAL_SERVER_ERROR.value(), t.getMessage(), reqId, ex.getStepContext()); - return resp.writeWith(Mono.just(resp.bufferFactory().wrap(JacksonUtils.writeValueAsString(rs).getBytes()))); - }else { - rs = new RespEntity(HttpStatus.INTERNAL_SERVER_ERROR.value(), t.getMessage(), reqId); - return resp.writeWith(Mono.just(resp.bufferFactory().wrap(rs.toString().getBytes()))); + if (t instanceof ExecuteScriptException) { + ExecuteScriptException ex = (ExecuteScriptException) t; + ServerHttpResponse resp = exchange.getResponse(); + resp.getHeaders().add(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON_VALUE); + RespEntity rs = null; + String reqId = exchange.getRequest().getId(); + if (ex.getStepContext() != null && ex.getStepContext().returnContext()) { + rs = new RespEntity(HttpStatus.INTERNAL_SERVER_ERROR.value(), t.getMessage(), reqId, ex.getStepContext()); + return resp.writeWith(Mono.just(resp.bufferFactory().wrap(JacksonUtils.writeValueAsString(rs).getBytes()))); + } else { + rs = new RespEntity(HttpStatus.INTERNAL_SERVER_ERROR.value(), t.getMessage(), reqId); + return resp.writeWith(Mono.just(resp.bufferFactory().wrap(rs.toString().getBytes()))); } } - Mono vm = WebUtils.responseError(exchange, filterExceptionHandler, HttpStatus.INTERNAL_SERVER_ERROR.value(), t.getMessage(), t); - return vm; + Mono vm = WebUtils.responseError(exchange, filterExceptionHandler, HttpStatus.INTERNAL_SERVER_ERROR.value(), t.getMessage(), t); + return vm; } } diff --git a/src/main/java/we/filter/FizzGatewayFilter.java b/src/main/java/we/filter/FizzGatewayFilter.java index 6738a20..31d22d2 100644 --- a/src/main/java/we/filter/FizzGatewayFilter.java +++ b/src/main/java/we/filter/FizzGatewayFilter.java @@ -24,9 +24,11 @@ import java.util.Map; import javax.annotation.Resource; +import com.alibaba.nacos.api.config.annotation.NacosValue; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; import org.springframework.core.annotation.Order; import org.springframework.core.io.buffer.DataBuffer; import org.springframework.core.io.buffer.DataBufferUtils; @@ -52,6 +54,7 @@ import we.fizz.ConfigLoader; import we.fizz.Pipeline; import we.fizz.input.Input; import we.flume.clients.log4j2appender.LogService; +import we.plugin.auth.ApiConfig; import we.util.Constants; import we.util.MapUtil; import we.util.WebUtils; @@ -66,21 +69,27 @@ public class FizzGatewayFilter implements WebFilter { private static final Logger LOGGER = LoggerFactory.getLogger(FizzGatewayFilter.class); private static final DataBuffer emptyBody = new NettyDataBufferFactory(new UnpooledByteBufAllocator(false, true)).wrap(Constants.Symbol.EMPTY.getBytes()); - + @Resource private ConfigLoader configLoader; - + + @NacosValue(value = "${need-auth:false}", autoRefreshed = true) + @Value("${need-auth:false}") + private boolean needAuth; + @Override public Mono filter(ServerWebExchange exchange, WebFilterChain chain) { + + String serviceId = WebUtils.getServiceId(exchange); + if (serviceId == null || (ApiConfig.Type.SERVICE_ARRANGE != WebUtils.getApiConfigType(exchange) && needAuth) ) { + return chain.filter(exchange); + } + long start = System.currentTimeMillis(); ServerHttpRequest request = exchange.getRequest(); ServerHttpResponse serverHttpResponse = exchange.getResponse(); - - if (WebUtils.getServiceId(exchange) == null) { - return chain.filter(exchange); - } - - String path = WebUtils.getPathPrefix(exchange) + WebUtils.getServiceId(exchange) + WebUtils.getReqPath(exchange); + + String path = WebUtils.getPathPrefix(exchange) + serviceId + WebUtils.getReqPath(exchange); String method = request.getMethodValue(); AggregateResource aggregateResource = configLoader.matchAggregateResource(method, path); if (aggregateResource == null) { @@ -95,7 +104,7 @@ public class FizzGatewayFilter implements WebFilter { if(fizzHeaders != null && !fizzHeaders.isEmpty()) { headers.putAll(fizzHeaders); } - + // traceId String tmpTraceId = CommonConstants.TRACE_ID_PREFIX + exchange.getRequest().getId(); if (StringUtils.isNotBlank(request.getHeaders().getFirst(CommonConstants.HEADER_TRACE_ID))) { @@ -112,8 +121,8 @@ public class FizzGatewayFilter implements WebFilter { clientInput.put("method", method); clientInput.put("headers", headers); clientInput.put("params", MapUtil.toHashMap(request.getQueryParams())); - - + + Mono result = null; if (HttpMethod.POST.name().equalsIgnoreCase(method)) { result = DataBufferUtils.join(request.getBody()).defaultIfEmpty(emptyBody).flatMap(buf -> { @@ -145,7 +154,7 @@ public class FizzGatewayFilter implements WebFilter { if (headerTraceIds == null || !headerTraceIds.contains(traceId)) { serverHttpResponse.getHeaders().add(CommonConstants.HEADER_TRACE_ID, traceId); } - + long end = System.currentTimeMillis(); pipeline.getStepContext().addElapsedTime("总耗时", end - start); LOGGER.info("ElapsedTimes={}", JSON.toJSONString(pipeline.getStepContext().getElapsedTimes())); diff --git a/src/main/java/we/filter/RouteFilter.java b/src/main/java/we/filter/RouteFilter.java index 1c8581f..e0860e9 100644 --- a/src/main/java/we/filter/RouteFilter.java +++ b/src/main/java/we/filter/RouteFilter.java @@ -22,6 +22,7 @@ import org.slf4j.LoggerFactory; import org.springframework.core.Ordered; import org.springframework.core.annotation.Order; import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpMethod; import org.springframework.http.HttpStatus; import org.springframework.http.server.reactive.ServerHttpRequest; import org.springframework.http.server.reactive.ServerHttpResponse; @@ -36,6 +37,7 @@ import we.legacy.RespEntity; import we.plugin.auth.ApiConfig; import we.plugin.auth.AuthPluginFilter; import we.proxy.FizzWebClient; +import we.util.Constants; import we.util.ThreadContext; import we.util.WebUtils; @@ -104,26 +106,28 @@ public class RouteFilter extends ProxyAggrFilter { ); } - ApiConfig ac = null; - Object authRes = WebUtils.getFilterResultDataItem(exchange, AuthPluginFilter.AUTH_PLUGIN_FILTER, AuthPluginFilter.RESULT); - if (authRes instanceof ApiConfig) { - ac = (ApiConfig) authRes; - } + String reqPath = WebUtils.getReqPath(exchange); + String rid = clientReq.getId(); - String relativeUri = WebUtils.getRelativeUri(exchange); - if (ac == null || ac.proxyMode == ApiConfig.DIRECT_PROXY_MODE) { + ApiConfig ac = WebUtils.getApiConfig(exchange); + if (ac == null) { + String relativeUri = WebUtils.getRelativeUri(exchange); return send(exchange, WebUtils.getServiceId(exchange), relativeUri, hdrs); + + } else if (ac.type == ApiConfig.Type.SERVICE_DISCOVERY) { + String relativeUri = WebUtils.appendQuery(ac.transform(reqPath), exchange); + return send(exchange, ac.backendService, relativeUri, hdrs); + + } else if (ac.type == ApiConfig.Type.REVERSE_PROXY) { + String relativeUri = ac.getNextHttpHostPort() + WebUtils.appendQuery(ac.transform(reqPath), exchange); + return fizzWebClient.send(rid, clientReq.getMethod(), relativeUri, hdrs, clientReq.getBody()).flatMap(genServerResponse(exchange)); + } else { - String realUri; - String backendUrl = ac.getNextBackendUrl(); - int acpLen = ac.path.length(); - if (acpLen == 1) { - realUri = backendUrl + relativeUri; - } else { - realUri = backendUrl + relativeUri.substring(acpLen); - } - relativeUri.substring(acpLen); - return fizzWebClient.send(clientReq.getId(), clientReq.getMethod(), realUri, hdrs, clientReq.getBody()).flatMap(genServerResponse(exchange)); + String err = "cant handle api config type " + ac.type; + StringBuilder b = ThreadContext.getStringBuilder(); + WebUtils.request2stringBuilder(exchange, b); + log.error(b.append(Constants.Symbol.LF).append(err).toString(), LogService.BIZ_ID, rid); + return WebUtils.buildJsonDirectResponseAndBindContext(exchange, HttpStatus.OK, null, RespEntity.toJson(HttpStatus.INTERNAL_SERVER_ERROR.value(), err, rid)); } } diff --git a/src/main/java/we/legacy/RespEntity.java b/src/main/java/we/legacy/RespEntity.java index a85d7af..a3155b4 100644 --- a/src/main/java/we/legacy/RespEntity.java +++ b/src/main/java/we/legacy/RespEntity.java @@ -32,7 +32,7 @@ public class RespEntity { private static final String f1 = ",\"message\":\""; private static final String f2 = "\"}"; - public int msgCode; + public int msgCode; public String message; diff --git a/src/main/java/we/plugin/auth/ApiConfig.java b/src/main/java/we/plugin/auth/ApiConfig.java index ba00d58..21b3798 100644 --- a/src/main/java/we/plugin/auth/ApiConfig.java +++ b/src/main/java/we/plugin/auth/ApiConfig.java @@ -18,16 +18,17 @@ package we.plugin.auth; import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; import org.apache.commons.lang3.StringUtils; import org.springframework.http.HttpMethod; - import we.plugin.PluginConfig; -import we.util.Constants; import we.util.JacksonUtils; +import we.util.UrlTransformUtils; -import java.util.*; +import java.util.Arrays; +import java.util.List; +import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLong; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -37,15 +38,20 @@ import java.util.stream.Stream; public class ApiConfig { - public static final int DELETED = 1; + public static interface Type { + static final byte UNDEFINED = 0; + static final byte SERVICE_ARRANGE = 1; + static final byte SERVICE_DISCOVERY = 2; + static final byte REVERSE_PROXY = 3; + } - public static final char ALLOW = 'a'; + public static final int DELETED = 1; - public static final char FORBID = 'f'; + public static final char ALLOW = 'a'; - public static final byte DIRECT_PROXY_MODE = 1; + public static final char FORBID = 'f'; - public static final byte PREFIX_REWRITE_PROXY_MODE = 2; + private static final String match_all = "/**"; // @JsonIgnore public int id; // tb_api_auth.id @@ -57,22 +63,50 @@ public class ApiConfig { public String service; + public String backendService; + public HttpMethod method = HttpMethod.X; - public String path = String.valueOf(Constants.Symbol.FORWARD_SLASH); +// public String path = String.valueOf(Constants.Symbol.FORWARD_SLASH); + public String path = match_all; + + public boolean exactMatch = false; + + public String backendPath; public Set apps = Stream.of(App.ALL_APP).collect(Collectors.toSet()); - public byte proxyMode = DIRECT_PROXY_MODE; + @JsonProperty("proxyMode") + public byte type = Type.SERVICE_DISCOVERY; private AtomicInteger counter = new AtomicInteger(-1); - public List backendUrls; +// public List backendUrls; + + public List httpHostPorts; public char access = ALLOW; public List pluginConfigs; + public static boolean isAntPathPattern(String path) { + boolean uriVar = false; + for (int i = 0; i < path.length(); i++) { + char c = path.charAt(i); + if (c == '*' || c == '?') { + return true; + } + if (c == '{') { + uriVar = true; + continue; + } + if (c == '}' && uriVar) { + return true; + } + } + return false; + } + public void setGatewayGroup(String ggs) { gatewayGroups.remove(GatewayGroup.DEFAULT); if (StringUtils.isBlank(ggs)) { @@ -101,7 +135,16 @@ public class ApiConfig { public void setPath(String p) { if (StringUtils.isNotBlank(p)) { - path = p.trim(); + if ("/".equals(p)) { + path = match_all; + } else { + path = p.trim(); + if (!isAntPathPattern(path)) { + exactMatch = true; + } + } + } else { + path = match_all; } } @@ -112,14 +155,31 @@ public class ApiConfig { } } + // @JsonIgnore + // public String getNextBackendUrl() { + // int idx = counter.incrementAndGet(); + // if (idx < 0) { + // counter.set(0); + // idx = 0; + // } + // return backendUrls.get(idx % backendUrls.size()); + // } + @JsonIgnore - public String getNextBackendUrl() { + public String getNextHttpHostPort() { int idx = counter.incrementAndGet(); if (idx < 0) { counter.set(0); idx = 0; } - return backendUrls.get(idx % backendUrls.size()); + return httpHostPorts.get(idx % httpHostPorts.size()); + } + + public String transform(String reqPath) { + if (exactMatch) { + return backendPath; + } + return UrlTransformUtils.transform(path, backendPath, reqPath); } @Override diff --git a/src/main/java/we/plugin/auth/ApiConfigService.java b/src/main/java/we/plugin/auth/ApiConfigService.java index 31e6b8d..7cdb696 100644 --- a/src/main/java/we/plugin/auth/ApiConfigService.java +++ b/src/main/java/we/plugin/auth/ApiConfigService.java @@ -53,16 +53,20 @@ public class ApiConfigService { private static final Logger log = LoggerFactory.getLogger(ApiConfigService.class); - private static final String fizzApiConfig = "fizz_api_config"; - - private static final String fizzApiConfigChannel = "fizz_api_config_channel"; - private static final String signHeader = "fizz-sign"; private static final String timestampHeader = "fizz-ts"; private static final String secretKeyHeader = "fizz-secretkey"; + @NacosValue(value = "${fizz-api-config.key:fizz_api_config_route}", autoRefreshed = true) + @Value("${fizz-api-config.key:fizz_api_config_route}") + private String fizzApiConfig; + + @NacosValue(value = "${fizz-api-config.channel:fizz_api_config_channel_route}", autoRefreshed = true) + @Value("${fizz-api-config.channel:fizz_api_config_channel_route}") + private String fizzApiConfigChannel; + public Map serviceConfigMap = new HashMap<>(128); private Map apiConfigMap = new HashMap<>(128); @@ -109,9 +113,9 @@ public class ApiConfigService { } } - @NacosValue(value = "${auth.compatible-wh:false}", autoRefreshed = true) - @Value("${auth.compatible-wh:false}") - private boolean compatibleWh; + @NacosValue(value = "${need-auth:false}", autoRefreshed = true) + @Value("${need-auth:false}") + private boolean needAuth; @Resource(name = AggregateRedisConfig.AGGREGATE_REACTIVE_REDIS_TEMPLATE) private ReactiveStringRedisTemplate rt; @@ -289,7 +293,7 @@ public class ApiConfigService { } ServiceConfig sc = serviceConfigMap.get(service); if (sc == null) { - if (compatibleWh) { + if (!needAuth) { return Mono.just(Access.YES); } else { return logWarnAndResult(service + Constants.Symbol.BLANK + Access.NO_SERVICE_CONFIG.getReason(), Access.NO_SERVICE_CONFIG); @@ -305,7 +309,7 @@ public class ApiConfigService { } ApiConfig ac = ac0; if (ac == null) { - if (compatibleWh) { + if (needAuth) { return Mono.just(Access.YES); } else { return logWarnAndResult(api + " no api config", Access.NO_API_CONFIG); diff --git a/src/main/java/we/plugin/auth/GatewayGroup2appsToApiConfig.java b/src/main/java/we/plugin/auth/GatewayGroup2appsToApiConfig.java index 91ff728..504f503 100644 --- a/src/main/java/we/plugin/auth/GatewayGroup2appsToApiConfig.java +++ b/src/main/java/we/plugin/auth/GatewayGroup2appsToApiConfig.java @@ -51,7 +51,7 @@ public class GatewayGroup2appsToApiConfig { } for (String a : ac.apps) { app2apiConfigMap.put(a, ac); - log.info(gg + " add " + a + " -> " + ac); + log.info("expose " + ac + " to " + gg + " group and " + a + " app"); } } } @@ -62,7 +62,7 @@ public class GatewayGroup2appsToApiConfig { if (app2apiConfigMap != null) { for (String a : ac.apps) { ApiConfig r = app2apiConfigMap.remove(a); - log.info(gg + " remove " + a + " -> " + r); + log.info("remove " + r + " from " + gg + " group and " + a + " app"); } } } @@ -77,7 +77,7 @@ public class GatewayGroup2appsToApiConfig { } for (String a : ac.apps) { ApiConfig old = app2apiConfigMap.put(a, ac); - log.info(gg + " update " + a + " -> " + old + " with " + ac); + log.info(gg + " group and " + a + " app update " + old + " with " + ac); } } } diff --git a/src/main/java/we/plugin/auth/ServiceConfig.java b/src/main/java/we/plugin/auth/ServiceConfig.java index 594a611..37c51ba 100644 --- a/src/main/java/we/plugin/auth/ServiceConfig.java +++ b/src/main/java/we/plugin/auth/ServiceConfig.java @@ -20,15 +20,14 @@ package we.plugin.auth; import com.fasterxml.jackson.annotation.JsonIgnore; import org.apache.commons.lang3.StringUtils; -import we.util.Constants; +import org.springframework.util.AntPathMatcher; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.http.HttpMethod; +import we.util.ThreadContext; -import java.util.EnumMap; -import java.util.HashMap; -import java.util.Map; +import java.util.*; /** * @author hongqiaowei @@ -36,7 +35,11 @@ import java.util.Map; public class ServiceConfig { - private static final Logger log = LoggerFactory.getLogger(ServiceConfig.class); + private static final Logger log = LoggerFactory.getLogger(ServiceConfig.class); + + private static final AntPathMatcher antPathMatcher = new AntPathMatcher(); + + private static final String mpps = "$mpps"; public String id; @@ -110,7 +113,8 @@ public class ServiceConfig { @JsonIgnore public ApiConfig getApiConfig(HttpMethod method, String path, String gatewayGroup, String app) { - GatewayGroup2appsToApiConfig r = getApiConfig0(method, path); +// GatewayGroup2appsToApiConfig r = getApiConfig0(method, path); + GatewayGroup2appsToApiConfig r = getApiConfig(method, path); if (r == null) { return null; } @@ -120,28 +124,57 @@ public class ServiceConfig { return r.get(gatewayGroup, app); } - private GatewayGroup2appsToApiConfig getApiConfig0(HttpMethod method, String path) { - while (true) { - EnumMap method2apiConfigMap = path2methodToApiConfigMapMap.get(path); - if (method2apiConfigMap == null) { - int i = path.lastIndexOf(Constants.Symbol.FORWARD_SLASH); - if (i == 0) { - method2apiConfigMap = path2methodToApiConfigMapMap.get(Constants.Symbol.FORWARD_SLASH_STR); - if (method2apiConfigMap == null) { - return null; - } else { - return getApiConfig1(method, method2apiConfigMap); - } - } else { - path = path.substring(0, i); + private GatewayGroup2appsToApiConfig getApiConfig(HttpMethod method, String reqPath) { + + List matchPathPatterns = ThreadContext.getArrayList(mpps, String.class); + + Set>> es = path2methodToApiConfigMapMap.entrySet(); + for (Map.Entry> e : es) { + String pathPattern = e.getKey(); + if (ApiConfig.isAntPathPattern(pathPattern)) { + if (antPathMatcher.match(pathPattern, reqPath)) { + matchPathPatterns.add(pathPattern); } - } else { - return getApiConfig1(method, method2apiConfigMap); + } else if (reqPath.equals(pathPattern)) { + return getGatewayGroup2appsToApiConfig(method, e.getValue()); } } + if (matchPathPatterns.isEmpty()) { + return null; + } else { + Collections.sort(matchPathPatterns, antPathMatcher.getPatternComparator(reqPath)); + String bestPattern = matchPathPatterns.get(0); + if (log.isDebugEnabled()) { + log.debug("\nreq path: " + reqPath + + "\nmatch patterns: " + matchPathPatterns + + "\nbest one: " + bestPattern); + } + return getGatewayGroup2appsToApiConfig(method, path2methodToApiConfigMapMap.get(bestPattern)); + } } - private GatewayGroup2appsToApiConfig getApiConfig1(HttpMethod method, EnumMap method2apiConfigMap) { + // private GatewayGroup2appsToApiConfig getApiConfig0(HttpMethod method, String path) { + // while (true) { + // EnumMap method2apiConfigMap = path2methodToApiConfigMapMap.get(path); + // if (method2apiConfigMap == null) { + // int i = path.lastIndexOf(Constants.Symbol.FORWARD_SLASH); + // if (i == 0) { + // method2apiConfigMap = path2methodToApiConfigMapMap.get(Constants.Symbol.FORWARD_SLASH_STR); + // if (method2apiConfigMap == null) { + // return null; + // } else { + // return getGatewayGroup2appsToApiConfig(method, method2apiConfigMap); + // } + // } else { + // path = path.substring(0, i); + // } + // } else { + // return getGatewayGroup2appsToApiConfig(method, method2apiConfigMap); + // } + // } + // } + + private GatewayGroup2appsToApiConfig getGatewayGroup2appsToApiConfig(HttpMethod method, EnumMap method2apiConfigMap) { GatewayGroup2appsToApiConfig r = method2apiConfigMap.get(method); if (r == null) { return method2apiConfigMap.get(HttpMethod.X); diff --git a/src/main/java/we/plugin/stat/StatPluginFilter.java b/src/main/java/we/plugin/stat/StatPluginFilter.java index c682faf..499a58f 100644 --- a/src/main/java/we/plugin/stat/StatPluginFilter.java +++ b/src/main/java/we/plugin/stat/StatPluginFilter.java @@ -105,7 +105,12 @@ public class StatPluginFilter extends PluginFilter { b.append(ip); toJsonStringValue(b, WebUtils.getOriginIp(exchange)); b.append(Constants.Symbol.COMMA); b.append(gatewayGroup); toJsonStringValue(b, currentGatewayGroups); b.append(Constants.Symbol.COMMA); b.append(service); toJsonStringValue(b, WebUtils.getServiceId(exchange)); b.append(Constants.Symbol.COMMA); - b.append(appid); toJsonStringValue(b, WebUtils.getAppId(exchange)); b.append(Constants.Symbol.COMMA); + + String appId = WebUtils.getAppId(exchange); + if (appId != null) { + b.append(appid); toJsonStringValue(b, appId); b.append(Constants.Symbol.COMMA); + } + b.append(apiMethod); toJsonStringValue(b, exchange.getRequest().getMethodValue()); b.append(Constants.Symbol.COMMA); b.append(apiPath); toJsonStringValue(b, WebUtils.getReqPath(exchange)); b.append(Constants.Symbol.COMMA); b.append(reqTime) .append(System.currentTimeMillis()); diff --git a/src/main/java/we/proxy/EurekaUriSelector.java b/src/main/java/we/proxy/EurekaUriSelector.java index a355549..80c489d 100644 --- a/src/main/java/we/proxy/EurekaUriSelector.java +++ b/src/main/java/we/proxy/EurekaUriSelector.java @@ -7,7 +7,9 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Service; import javax.annotation.Resource; +import java.util.ArrayList; import java.util.List; +import java.util.concurrent.atomic.AtomicLong; /** * The Eureka implementation of {@code DiscoveryClientUriSelector} @@ -30,11 +32,11 @@ public class EurekaUriSelector extends AbstractDiscoveryClientUriSelector { // private static List aggrMemberInsts = new ArrayList<>(); // static { - // InstanceInfo i0 = InstanceInfo.Builder.newBuilder().setAppName("TRIP-MINI").setIPAddr("xxx.25.63.192").setPort(7094).build(); + // InstanceInfo i0 = InstanceInfo.Builder.newBuilder().setAppName("MINITRIP").setIPAddr("xxx.xxx.63.192").setPort(7094).build(); // aggrMemberInsts.add(i0); // } // private static AtomicLong counter = new AtomicLong(0); - // private static final String aggrMember = "trip-mini"; + // private static final String aggrMember = "minitrip"; private InstanceInfo roundRobinChoose1instFrom(String service) { diff --git a/src/main/java/we/proxy/FizzWebClient.java b/src/main/java/we/proxy/FizzWebClient.java index 90a550e..c9e8cac 100644 --- a/src/main/java/we/proxy/FizzWebClient.java +++ b/src/main/java/we/proxy/FizzWebClient.java @@ -18,6 +18,7 @@ package we.proxy; import com.alibaba.nacos.api.config.annotation.NacosValue; +import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Value; @@ -219,14 +220,13 @@ public class FizzWebClient { // TODO 请求完成后，做metric, 以反哺后续的请求转发 } - - private String extractServiceOrAddress(String uriOrSvc) { return uriOrSvc.substring(7, uriOrSvc.indexOf(Constants.Symbol.FORWARD_SLASH, 10)); } private boolean isService(String s) { - if (s.indexOf(Constants.Symbol.DOT) > 0 || s.equals(localhost)) { + if (StringUtils.indexOfAny(s, Constants.Symbol.DOT, Constants.Symbol.COLON) > 0 + || StringUtils.indexOfIgnoreCase(s, localhost) > 0) { return false; } else { return true; diff --git a/src/main/java/we/util/Constants.java b/src/main/java/we/util/Constants.java index b17404f..ae3100a 100644 --- a/src/main/java/we/util/Constants.java +++ b/src/main/java/we/util/Constants.java @@ -28,11 +28,12 @@ public final class Constants { public static final class Symbol { public static final String EMPTY = ""; public static final String SPACE_STR = " "; - public static final String TWO_SPACE_STR = " "; - public static final char COMMA = ','; - public static final char COLON = ':'; public static final char BLANK = ' '; public static final char SPACE = BLANK; + public static final String TWO_SPACE_STR = " "; + + public static final char COMMA = ','; + public static final char COLON = ':'; public static final char FORWARD_SLASH = '/'; public static final String FORWARD_SLASH_STR = "/"; public static final char BACK_SLASH = '\\'; @@ -51,15 +52,17 @@ public final class Constants { public static final char RIGHT_SQUARE_BRACKET = ']'; public static final char LEFT_BRACE = '{'; public static final char RIGHT_BRACE = '}'; - public static final String LINE_SEPARATOR = System.lineSeparator(); - public static final String COMMA_SPACE = ", "; + public static final char LF = '\n'; public static final char TAB = '\t'; public static final char NUL = '\u0000'; - public static final String HTTP_PROTOCOL_PREFIX = "http://"; static final char c0 = SystemUtils.IS_OS_WINDOWS ? Constants.Symbol.BACK_SLASH : Constants.Symbol.FORWARD_SLASH; public static final char PATH_SEPARATOR = c0; + public static final String LINE_SEPARATOR = System.lineSeparator(); + + public static final String COMMA_SPACE = ", "; + public static final String HTTP_PROTOCOL_PREFIX = "http://"; } public static final class Charset { @@ -79,22 +82,28 @@ public final class Constants { public static final class Profiles { public static final String LOCAL = "local"; public static final String DEV = "dev"; - public static final String TEST = "test"; - public static final String PREPROD = "preprod"; - public static final String PRE = "pre"; - public static final String PROD = "prod"; - public static final String HTTP_SERVER = "http_server"; - public static final String HTTP_CLIENT = "http_client"; - public static final String MYSQL = "mysql"; - public static final String REDIS = "redis"; - public static final String CODIS = "codis"; - public static final String MONGO = "mongo"; - public static final String ACTIVEMQ = "activemq"; - public static final String KAFKA = "kafka"; - public static final String ELASTICSEARCH = "elasticsearch"; - public static final String SCHED = "sched"; + public static final String TEST = "test"; + public static final String FAT = "fat"; + + public static final String PREPROD = "preprod"; + public static final String UAT = "uat"; + public static final String PRE = "pre"; + + public static final String PROD = "prod"; + public static final String PRO = "pro"; } + public static final String HTTP_SERVER = "http_server"; + public static final String HTTP_CLIENT = "http_client"; + public static final String MYSQL = "mysql"; + public static final String REDIS = "redis"; + public static final String CODIS = "codis"; + public static final String MONGO = "mongo"; + public static final String ACTIVEMQ = "activemq"; + public static final String KAFKA = "kafka"; + public static final String ELASTICSEARCH = "elasticsearch"; + public static final String SCHED = "sched"; + public static final String BIZ_ID = "bizId"; } diff --git a/src/main/java/we/util/ThreadContext.java b/src/main/java/we/util/ThreadContext.java index f654f47..86565eb 100644 --- a/src/main/java/we/util/ThreadContext.java +++ b/src/main/java/we/util/ThreadContext.java @@ -18,6 +18,7 @@ package we.util; import java.text.SimpleDateFormat; +import java.util.ArrayList; import java.util.HashMap; import java.util.Map; @@ -111,4 +112,19 @@ public abstract class ThreadContext { public static Object remove(String key) { return getMap().remove(key); } + + public static ArrayList getArrayList(String key, Class elementType) { + return getArrayList(key, elementType, true); + } + + public static ArrayList getArrayList(String key, Class elementType, boolean clear) { + ArrayList l = (ArrayList) get(key); + if (l == null) { + l = new ArrayList<>(); + set(key, l); + } else if (clear) { + l.clear(); + } + return l; + } } diff --git a/src/main/java/we/util/UrlTransformUtils.java b/src/main/java/we/util/UrlTransformUtils.java new file mode 100644 index 0000000..6891e89 --- /dev/null +++ b/src/main/java/we/util/UrlTransformUtils.java @@ -0,0 +1,307 @@ +package we.util; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.lang.Nullable; +import org.springframework.util.AntPathMatcher; +import org.springframework.util.Assert; +import we.filter.RouteFilter; + +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Fizz gateway url transform util class + * + * @author zhongjie + */ +public class UrlTransformUtils { + + private static final Logger log = LoggerFactory.getLogger(UrlTransformUtils.class); + + private UrlTransformUtils() {} + + private static final FizzGatewayUrlAntPathMatcher ANT_PATH_MATCHER = new FizzGatewayUrlAntPathMatcher(); + + /** + * transform the backend path to the real backend request path + * @param frontendPath frontend path + * @param backendPath backend path + * @param reqPath request path + * @return the transformed backend path + * @throws IllegalStateException when the request path does not match the frontend path pattern + * @throws IllegalArgumentException The number of capturing groups in the pattern segment does not match the number of URI template variables it defines + */ + public static String transform(String frontendPath, String backendPath, String reqPath) { + Assert.hasText(frontendPath, "frontend path cannot be null"); + Assert.hasText(backendPath, "backend path cannot be null"); + Assert.hasText(reqPath, "req path cannot be null"); + String bp = backendPath; + Map variables = ANT_PATH_MATCHER.extractUriTemplateVariables(frontendPath, reqPath); + for (Map.Entry entry : variables.entrySet()) { + backendPath = backendPath.replaceAll("\\{" + Matcher.quoteReplacement(entry.getKey()) + "}", entry.getValue()); + } + + if (backendPath.indexOf('{') != -1) { + backendPath = backendPath.replaceAll("\\{[^/]*}", ""); + } + + if (log.isDebugEnabled()) { + log.debug("req: " + reqPath + ", frontend: " + frontendPath + ", backend: " + bp + ", target: " + backendPath); + } + + return backendPath; + } + + /** + * 自定义Ant风格路径匹配器 + * 设置默认路径分隔符为{@code #} + * 使用{@link FizzGatewayUrlAntPathMatcher.FizzGatewayAntPathStringMatcher}设置自定义的参数变量值（额外返回变量名为$1...n的键值对） + * + * @author zhongjie + */ + static class FizzGatewayUrlAntPathMatcher extends AntPathMatcher { + private static final String DEFAULT_PATH_SEPARATOR = "#"; + + private static final int CACHE_TURNOFF_THRESHOLD = 65536; + + private volatile Boolean cachePatterns; + + private final Map replaceDoubleStarPatternCache = new ConcurrentHashMap<>(256); + + private final Map tokenizedPatternCache = new ConcurrentHashMap<>(256); + + final Map stringMatcherCache = new ConcurrentHashMap<>(256); + + private boolean caseSensitive = true; + + private static AntPathMatcher DEFAULT_ANT_PATH_MATCHER = new AntPathMatcher(); + + public FizzGatewayUrlAntPathMatcher() { + // 设置默认路径分隔符为# + super(DEFAULT_PATH_SEPARATOR); + } + + @Override + public void setPathSeparator(String pathSeparator) { + throw new RuntimeException("operation not support"); + } + + @Override + public void setTrimTokens(boolean trimTokens) { + throw new RuntimeException("operation not support"); + } + + @Override + public void setCaseSensitive(boolean caseSensitive) { + super.setCaseSensitive(caseSensitive); + this.caseSensitive = caseSensitive; + } + + @Override + public void setCachePatterns(boolean cachePatterns) { + super.setCachePatterns(cachePatterns); + this.cachePatterns = cachePatterns; + } + + @Override + protected AntPathStringMatcher getStringMatcher(String pattern) { + AntPathStringMatcher matcher = null; + Boolean cachePatterns = this.cachePatterns; + if (cachePatterns == null || cachePatterns) { + matcher = this.stringMatcherCache.get(pattern); + } + if (matcher == null) { + matcher = new FizzGatewayAntPathStringMatcher(pattern, this.caseSensitive); + if (cachePatterns == null && this.stringMatcherCache.size() >= CACHE_TURNOFF_THRESHOLD) { + // Try to adapt to the runtime situation that we're encountering: + // There are obviously too many different patterns coming in here... + // So let's turn off the cache since the patterns are unlikely to be reoccurring. + deactivatePatternCache(); + return matcher; + } + if (cachePatterns == null || cachePatterns) { + this.stringMatcherCache.put(pattern, matcher); + } + } + return matcher; + } + + @Override + protected String[] tokenizePattern(String pattern) { + String[] tokenized = null; + Boolean cachePatterns = this.cachePatterns; + if (cachePatterns == null || cachePatterns) { + tokenized = this.tokenizedPatternCache.get(pattern); + } + if (tokenized == null) { + tokenized = tokenizePath(pattern); + if (cachePatterns == null && this.tokenizedPatternCache.size() >= CACHE_TURNOFF_THRESHOLD) { + // Try to adapt to the runtime situation that we're encountering: + // There are obviously too many different patterns coming in here... + // So let's turn off the cache since the patterns are unlikely to be reoccurring. + deactivatePatternCache(); + return tokenized; + } + if (cachePatterns == null || cachePatterns) { + this.tokenizedPatternCache.put(pattern, tokenized); + } + } + return tokenized; + } + + private void deactivatePatternCache() { + this.cachePatterns = false; + this.tokenizedPatternCache.clear(); + this.stringMatcherCache.clear(); + this.replaceDoubleStarPatternCache.clear(); + } + + @Override + public String extractPathWithinPattern(String pattern, String path) { + return DEFAULT_ANT_PATH_MATCHER.extractPathWithinPattern(pattern, path); + } + + @Override + public String combine(String pattern1, String pattern2) { + return DEFAULT_ANT_PATH_MATCHER.combine(pattern1, pattern2); + } + + @Override + protected boolean doMatch(String pattern, String path, boolean fullMatch, Map uriTemplateVariables) { + String replaceDoubleStarPattern = null; + if (pattern != null) { + replaceDoubleStarPattern = getReplaceDoubleStarPattern(pattern); + } + return super.doMatch(replaceDoubleStarPattern, path, fullMatch, uriTemplateVariables); + } + + private String getReplaceDoubleStarPattern(String pattern) { + String replaceDoubleStarPattern = null; + Boolean cachePatterns = this.cachePatterns; + if (cachePatterns == null || cachePatterns) { + replaceDoubleStarPattern = this.replaceDoubleStarPatternCache.get(pattern); + } + if (replaceDoubleStarPattern == null) { + // by-zhongjie 替换**为.*正则模式 + replaceDoubleStarPattern = pattern.replaceAll("/\\*\\*$", "/{\\$:.*}") + .replaceAll("/\\*\\*/", "/{\\$:.*}/") + .replaceAll("^\\*\\*/", "{\\$:.*}/"); + if (cachePatterns == null && this.replaceDoubleStarPatternCache.size() >= CACHE_TURNOFF_THRESHOLD) { + // Try to adapt to the runtime situation that we're encountering: + // There are obviously too many different patterns coming in here... + // So let's turn off the cache since the patterns are unlikely to be reoccurring. + deactivatePatternCache(); + return replaceDoubleStarPattern; + } + if (cachePatterns == null || cachePatterns) { + this.replaceDoubleStarPatternCache.put(pattern, replaceDoubleStarPattern); + } + } + return replaceDoubleStarPattern; + } + + protected static class FizzGatewayAntPathStringMatcher extends AntPathStringMatcher { + // by-zhongjie 将 \?|\*|\{((?:\{[^/]+?\}|[^/{}]|\\[{}])+?)\} 改为 \?|\*|\{((?:\{[^/]+?\}|[^{}]|\\[{}])+?)\}，排除/的限制 + private static final Pattern GLOB_PATTERN = Pattern.compile("\\?|\\*|\\{((?:\\{[^/]+?\\}|[^{}]|\\\\[{}])+?)\\}"); + + // by-zhongjie 将 (.*) 改为 ([^/]*)，限制变量只能匹配在非/的字符内 + private static final String DEFAULT_VARIABLE_PATTERN = "([^/]*)"; + + private final Pattern pattern; + + private final List variableNames = new LinkedList<>(); + + // by-zhongjie 匿名占位符 + private final String ANONYMOUS_PLACEHOLDER = "$"; + + public FizzGatewayAntPathStringMatcher(String pattern) { + this(pattern, true); + } + + public FizzGatewayAntPathStringMatcher(String pattern, boolean caseSensitive) { + super(pattern, caseSensitive); + StringBuilder patternBuilder = new StringBuilder(); + Matcher matcher = GLOB_PATTERN.matcher(pattern); + int end = 0; + while (matcher.find()) { + patternBuilder.append(quote(pattern, end, matcher.start())); + String match = matcher.group(); + if ("?".equals(match)) { + // by-zhongjie 对 ? 也使用模式匹配 + patternBuilder.append('('); + patternBuilder.append('.'); + patternBuilder.append(')'); + this.variableNames.add(ANONYMOUS_PLACEHOLDER); + } + else if ("*".equals(match)) { + // by-zhongjie 对 * 也使用模式匹配 + patternBuilder.append(DEFAULT_VARIABLE_PATTERN); + this.variableNames.add(ANONYMOUS_PLACEHOLDER); + } + else if (match.startsWith("{") && match.endsWith("}")) { + int colonIdx = match.indexOf(':'); + if (colonIdx == -1) { + patternBuilder.append(DEFAULT_VARIABLE_PATTERN); + this.variableNames.add(matcher.group(1)); + } + else { + String variablePattern = match.substring(colonIdx + 1, match.length() - 1); + patternBuilder.append('('); + patternBuilder.append(variablePattern); + patternBuilder.append(')'); + String variableName = match.substring(1, colonIdx); + this.variableNames.add(variableName); + } + } + end = matcher.end(); + } + patternBuilder.append(quote(pattern, end, pattern.length())); + this.pattern = (caseSensitive ? Pattern.compile(patternBuilder.toString()) : + Pattern.compile(patternBuilder.toString(), Pattern.CASE_INSENSITIVE)); + } + + private String quote(String s, int start, int end) { + if (start == end) { + return ""; + } + return Pattern.quote(s.substring(start, end)); + } + + + @Override + public boolean matchStrings(String str, @Nullable Map uriTemplateVariables) { + Matcher matcher = this.pattern.matcher(str); + if (matcher.matches()) { + if (uriTemplateVariables != null) { + // SPR-8455 + if (this.variableNames.size() != matcher.groupCount()) { + throw new IllegalArgumentException("The number of capturing groups in the pattern segment " + + this.pattern + " does not match the number of URI template variables it defines, " + + "which can occur if capturing groups are used in a URI template regex. " + + "Use non-capturing groups instead."); + } + for (int i = 1; i <= matcher.groupCount(); i++) { + String name = this.variableNames.get(i - 1); + String value = matcher.group(i); + + if (!ANONYMOUS_PLACEHOLDER.equals(name)) { + uriTemplateVariables.put(name, value); + } + // by-zhongjie 对提取到的变量按序号输出 + uriTemplateVariables.put(ANONYMOUS_PLACEHOLDER + i, value); + } + } + return true; + } + else { + return false; + } + } + } + } +} diff --git a/src/main/java/we/util/WebUtils.java b/src/main/java/we/util/WebUtils.java index 775d10d..023eeb8 100644 --- a/src/main/java/we/util/WebUtils.java +++ b/src/main/java/we/util/WebUtils.java @@ -33,7 +33,10 @@ import reactor.core.publisher.Mono; import we.filter.FilterResult; import we.flume.clients.log4j2appender.LogService; import we.legacy.RespEntity; +import we.plugin.auth.ApiConfig; +import we.plugin.auth.AuthPluginFilter; +import java.net.URI; import java.util.*; /** @@ -44,18 +47,6 @@ public abstract class WebUtils { private static final Logger log = LoggerFactory.getLogger(WebUtils.class); - public static final String APP_HEADER = "fizz-appid"; - - private static final String directResponse = "directResponse"; - - public static final String FILTER_CONTEXT = "filterContext"; - - public static final String APPEND_HEADERS = "appendHeaders"; - - public static final String PREV_FILTER_RESULT = "prevFilterResult"; - - public static final String request_path = "reqPath"; - private static final String SERVICE_ID = "serviceId"; private static final String xForwardedFor = "X-FORWARDED-FOR"; @@ -66,16 +57,28 @@ public abstract class WebUtils { private static final String binaryAddress = "0:0:0:0:0:0:0:1"; - public static boolean logResponseBody = false; - - public static Set logHeaderSet = Collections.EMPTY_SET; + private static final String directResponse = "directResponse"; private static final String response = " response "; private static final String originIp = "originIp"; - - public static final String PATH_PREFIX = "/proxy/"; + public static final String APP_HEADER = "fizz-appid"; + + public static final String FILTER_CONTEXT = "filterContext"; + + public static final String APPEND_HEADERS = "appendHeaders"; + + public static final String PREV_FILTER_RESULT = "prevFilterResult"; + + public static final String request_path = "reqPath"; + + public static boolean logResponseBody = false; + + public static Set logHeaderSet = Collections.EMPTY_SET; + + public static final String PATH_PREFIX = "/proxy/"; + public static String getHeaderValue(ServerWebExchange exchange, String header) { return exchange.getRequest().getHeaders().getFirst(header); } @@ -120,6 +123,24 @@ public abstract class WebUtils { } return svc; } + + public static byte getApiConfigType(ServerWebExchange exchange) { + ApiConfig ac = getApiConfig(exchange); + if (ac == null) { + return ApiConfig.Type.UNDEFINED; + } else { + return ac.type; + } + } + + public static ApiConfig getApiConfig(ServerWebExchange exchange) { + Object authRes = getFilterResultDataItem(exchange, AuthPluginFilter.AUTH_PLUGIN_FILTER, AuthPluginFilter.RESULT); + if (authRes != null && authRes instanceof ApiConfig) { + return (ApiConfig) authRes; + } else { + return null; + } + } public static String getPathPrefix(ServerWebExchange exchange) { String p = exchange.getRequest().getPath().value(); @@ -227,18 +248,34 @@ public abstract class WebUtils { return path; } - public static String getRelativeUri(ServerWebExchange exchange) { - String relativeUri = getReqPath(exchange); - String qry = exchange.getRequest().getURI().getQuery(); + public static String getQuery(ServerWebExchange exchange) { + URI uri = exchange.getRequest().getURI(); + String qry = uri.getQuery(); if (qry != null) { if (StringUtils.indexOfAny(qry, Constants.Symbol.LEFT_BRACE, Constants.Symbol.FORWARD_SLASH, Constants.Symbol.HASH) > 0) { - qry = exchange.getRequest().getURI().getRawQuery(); + qry = uri.getRawQuery(); } + } + return qry; + } + + public static String getRelativeUri(ServerWebExchange exchange) { + String relativeUri = getReqPath(exchange); + String qry = getQuery(exchange); + if (qry != null) { relativeUri = relativeUri + Constants.Symbol.QUESTION + qry; } return relativeUri; } + public static String appendQuery(String path, ServerWebExchange exchange) { + String qry = getQuery(exchange); + if (qry != null) { + return path + Constants.Symbol.QUESTION + qry; + } + return path; + } + public static Map getAppendHeaders(ServerWebExchange exchange) { return (Map) exchange.getAttributes().get(APPEND_HEADERS); }