222
This commit is contained in:
parent
ef2f3688c8
commit
02b9bdd556
@ -0,0 +1,205 @@
|
||||
package com.ruoyi.web.util;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.InputStream;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Random;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.yaml.snakeyaml.Yaml;
|
||||
|
||||
import us.codecraft.webmagic.downloader.HttpClientDownloader;
|
||||
import us.codecraft.webmagic.proxy.Proxy;
|
||||
import us.codecraft.webmagic.proxy.SimpleProxyProvider;
|
||||
|
||||
/**
|
||||
* WebMagic爬虫代理工具类
|
||||
*
|
||||
* @author ruoyi
|
||||
*/
|
||||
public class WebMagicProxyUtil {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(WebMagicProxyUtil.class);
|
||||
|
||||
// 随机数生成器
|
||||
private static final Random random = new Random();
|
||||
|
||||
// 默认代理配置
|
||||
private static final String DEFAULT_PROXY_HOST = "127.0.0.1";
|
||||
private static final String DEFAULT_PROXY_PORT = "7890";
|
||||
|
||||
// 存储所有代理节点信息
|
||||
private static List<Map<String, Object>> proxyNodes = new ArrayList<>();
|
||||
|
||||
// 当前使用的代理信息
|
||||
private static String currentProxyName = "";
|
||||
private static String currentProxyHost = DEFAULT_PROXY_HOST;
|
||||
private static int currentProxyPort = Integer.parseInt(DEFAULT_PROXY_PORT);
|
||||
|
||||
static {
|
||||
loadProxyConfig();
|
||||
}
|
||||
|
||||
/**
|
||||
* 加载代理配置文件
|
||||
*/
|
||||
@SuppressWarnings("unchecked")
|
||||
private static void loadProxyConfig() {
|
||||
try {
|
||||
// 首先尝试从config目录加载
|
||||
File configFile = new File("/www/java_mall/erp/config/test_proxy.yml");
|
||||
|
||||
// 如果配置文件不存在，尝试从类路径加载
|
||||
if (!configFile.exists()) {
|
||||
configFile = new File("config/test_proxy.yml");
|
||||
}
|
||||
|
||||
if (configFile.exists()) {
|
||||
Yaml yaml = new Yaml();
|
||||
try (InputStream inputStream = new FileInputStream(configFile)) {
|
||||
Map<String, Object> config = yaml.load(inputStream);
|
||||
if (config != null && config.containsKey("proxies")) {
|
||||
List<Map<String, Object>> proxies = (List<Map<String, Object>>) config.get("proxies");
|
||||
proxyNodes = proxies.stream()
|
||||
.filter(proxy -> proxy != null && proxy.containsKey("name") && proxy.containsKey("server") && proxy.containsKey("port"))
|
||||
.collect(Collectors.toList());
|
||||
logger.info("成功加载{}个代理节点配置", proxyNodes.size());
|
||||
}
|
||||
}
|
||||
} else {
|
||||
logger.warn("未找到代理配置文件，将使用默认代理设置");
|
||||
}
|
||||
} catch (Exception e) {
|
||||
logger.error("加载代理配置失败", e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取所有可用代理节点
|
||||
*/
|
||||
public static List<Map<String, String>> getAllProxies() {
|
||||
if (proxyNodes.isEmpty()) {
|
||||
loadProxyConfig();
|
||||
}
|
||||
|
||||
if (proxyNodes.isEmpty()) {
|
||||
// 如果配置文件中没有代理，至少返回一个默认代理
|
||||
List<Map<String, String>> defaultProxyList = new ArrayList<>();
|
||||
Map<String, String> defaultProxy = new HashMap<>();
|
||||
defaultProxy.put("name", "默认代理");
|
||||
defaultProxy.put("server", DEFAULT_PROXY_HOST);
|
||||
defaultProxy.put("port", DEFAULT_PROXY_PORT);
|
||||
defaultProxy.put("type", "http");
|
||||
defaultProxyList.add(defaultProxy);
|
||||
return defaultProxyList;
|
||||
}
|
||||
|
||||
return proxyNodes.stream()
|
||||
.map(node -> {
|
||||
Map<String, String> proxyInfo = new HashMap<>();
|
||||
proxyInfo.put("name", (String) node.get("name"));
|
||||
proxyInfo.put("server", (String) node.get("server"));
|
||||
proxyInfo.put("port", String.valueOf(node.get("port")));
|
||||
proxyInfo.put("type", (String) node.get("type"));
|
||||
return proxyInfo;
|
||||
})
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
|
||||
/**
|
||||
* 设置当前使用的代理节点
|
||||
*/
|
||||
public static boolean setCurrentProxy(String proxyName) {
|
||||
// 查找匹配的代理节点
|
||||
Map<String, Object> targetProxy = proxyNodes.stream()
|
||||
.filter(node -> proxyName.equals(node.get("name")))
|
||||
.findFirst()
|
||||
.orElse(null);
|
||||
|
||||
if (targetProxy != null) {
|
||||
currentProxyName = (String) targetProxy.get("name");
|
||||
currentProxyHost = (String) targetProxy.get("server");
|
||||
currentProxyPort = ((Number) targetProxy.get("port")).intValue();
|
||||
|
||||
logger.info("已设置代理节点: {}, 地址: {}:{}", currentProxyName, currentProxyHost, currentProxyPort);
|
||||
return true;
|
||||
} else if ("默认代理".equals(proxyName)) {
|
||||
// 使用默认代理
|
||||
currentProxyName = "默认代理";
|
||||
currentProxyHost = DEFAULT_PROXY_HOST;
|
||||
currentProxyPort = Integer.parseInt(DEFAULT_PROXY_PORT);
|
||||
logger.info("已设置默认代理: {}:{}", DEFAULT_PROXY_HOST, DEFAULT_PROXY_PORT);
|
||||
return true;
|
||||
} else {
|
||||
logger.warn("未找到指定的代理节点: {}, 将使用默认代理", proxyName);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取配置了代理的下载器
|
||||
*/
|
||||
public static HttpClientDownloader getProxyDownloader() {
|
||||
return getProxyDownloader(currentProxyHost, currentProxyPort);
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取配置了指定代理的下载器
|
||||
*/
|
||||
public static HttpClientDownloader getProxyDownloader(String host, int port) {
|
||||
HttpClientDownloader httpClientDownloader = new HttpClientDownloader();
|
||||
clearSystemProxy();
|
||||
|
||||
try {
|
||||
// 设置系统代理
|
||||
System.setProperty("http.proxyHost", host);
|
||||
System.setProperty("http.proxyPort", String.valueOf(port));
|
||||
System.setProperty("https.proxyHost", host);
|
||||
System.setProperty("https.proxyPort", String.valueOf(port));
|
||||
System.setProperty("http.nonProxyHosts", "localhost|127.0.0.1");
|
||||
|
||||
// 设置WebMagic代理
|
||||
httpClientDownloader.setProxyProvider(SimpleProxyProvider.from(
|
||||
new Proxy(host, port)
|
||||
));
|
||||
|
||||
logger.info("已配置代理: {}:{}", host, port);
|
||||
} catch (Exception e) {
|
||||
logger.error("设置代理失败", e);
|
||||
}
|
||||
|
||||
return httpClientDownloader;
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取配置了默认代理的下载器
|
||||
*/
|
||||
public static HttpClientDownloader getDefaultProxyDownloader() {
|
||||
return getProxyDownloader(DEFAULT_PROXY_HOST, Integer.parseInt(DEFAULT_PROXY_PORT));
|
||||
}
|
||||
|
||||
/**
|
||||
* 清除系统代理设置
|
||||
*/
|
||||
public static void clearSystemProxy() {
|
||||
System.clearProperty("http.proxyHost");
|
||||
System.clearProperty("http.proxyPort");
|
||||
System.clearProperty("https.proxyHost");
|
||||
System.clearProperty("https.proxyPort");
|
||||
System.clearProperty("socksProxyHost");
|
||||
System.clearProperty("socksProxyPort");
|
||||
}
|
||||
|
||||
/**
|
||||
* 获取随机休眠时间
|
||||
*/
|
||||
public static int getRandomSleepTime(int min, int max) {
|
||||
return min + random.nextInt(max - min);
|
||||
}
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user