From 6b01a5fbfeda9af5720c2ecd127267a77bc399e7 Mon Sep 17 00:00:00 2001 From: ZiJIe <17738440858@163.com> Date: Tue, 15 Jul 2025 17:39:12 +0800 Subject: [PATCH] 222 --- ruoyi-admin/pom.xml | 10 +- .../web/controller/ProductsController.java | 17 -- .../controller/tool/ProductsController.java | 61 ++++++ .../controller/tool/RakutenController.java | 189 ++++++++++++++++++ .../controller/tool/WebMagicController.java | 14 +- ruoyi-ui/src/api/prod/ozon.js | 9 + ruoyi-ui/src/api/prod/rakuten.js | 9 + ruoyi-ui/src/views/prod/ozon/index.vue | 85 ++++++++ ruoyi-ui/src/views/prod/rakuten/index.vue | 110 ++++++++++ 9 files changed, 473 insertions(+), 31 deletions(-) delete mode 100644 ruoyi-admin/src/main/java/com/ruoyi/web/controller/ProductsController.java create mode 100644 ruoyi-admin/src/main/java/com/ruoyi/web/controller/tool/ProductsController.java create mode 100644 ruoyi-admin/src/main/java/com/ruoyi/web/controller/tool/RakutenController.java create mode 100644 ruoyi-ui/src/api/prod/ozon.js create mode 100644 ruoyi-ui/src/api/prod/rakuten.js create mode 100644 ruoyi-ui/src/views/prod/ozon/index.vue create mode 100644 ruoyi-ui/src/views/prod/rakuten/index.vue diff --git a/ruoyi-admin/pom.xml b/ruoyi-admin/pom.xml index 6c69ee70..6321c9b2 100644 --- a/ruoyi-admin/pom.xml +++ b/ruoyi-admin/pom.xml @@ -99,7 +99,15 @@ false ${project.artifactId} - + + + org.apache.maven.plugins + maven-compiler-plugin + + 15 + 15 + + ${project.artifactId} diff --git a/ruoyi-admin/src/main/java/com/ruoyi/web/controller/ProductsController.java b/ruoyi-admin/src/main/java/com/ruoyi/web/controller/ProductsController.java deleted file mode 100644 index d43dbbd5..00000000 --- a/ruoyi-admin/src/main/java/com/ruoyi/web/controller/ProductsController.java +++ /dev/null @@ -1,17 +0,0 @@ -package com.ruoyi.web.controller; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RestController; -import com.ruoyi.common.core.controller.BaseController; -/** - * prodController - * - * @author ruoyi - * @date 2025-06-21 - */ -@RestController -@RequestMapping("/prod/products") -public class ProductsController extends BaseController -{ - - -} diff --git a/ruoyi-admin/src/main/java/com/ruoyi/web/controller/tool/ProductsController.java b/ruoyi-admin/src/main/java/com/ruoyi/web/controller/tool/ProductsController.java new file mode 100644 index 00000000..884c117d --- /dev/null +++ b/ruoyi-admin/src/main/java/com/ruoyi/web/controller/tool/ProductsController.java @@ -0,0 +1,61 @@ +package com.ruoyi.web.controller.tool; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.bind.annotation.GetMapping; +import com.ruoyi.common.core.controller.BaseController; +import com.ruoyi.common.core.domain.AjaxResult; +import us.codecraft.webmagic.Page; +import us.codecraft.webmagic.Site; +import us.codecraft.webmagic.Spider; +import us.codecraft.webmagic.processor.PageProcessor; + +import java.util.ArrayList; +import java.util.List; + +/** + * 1688Controller + * + * @author ruoyi + * @date 2025-07-15 + */ +@RestController +@RequestMapping("/prod/ozon") +public class ProductsController extends BaseController +{ + @GetMapping("/scrapeImages") + public AjaxResult scrapeImages() + { + String url = "https://www.ozon.ru/highlight/ozon-global/?currency_price=14.000%3B500.000"; + + List imageUrls = new ArrayList<>(); + + Site site = Site.me() + .setRetryTimes(3) + .setTimeOut(10000); + site.addHeader("cookie", "xcid=b30f6db523d4aee734a822aa0a230b3f; __Secure-ext_xcid=b30f6db523d4aee734a822aa0a230b3f; __Secure-ab-group=92; rfuid=NjkyNDcyNDUyLDEyNC4wNDM0NzUyNzUxNjA3NCwxMDI4MjM3MjIzLC0xLC05ODc0NjQ3MjQsVzNzaWJtRnRaU0k2SWtOb2NtOXRhWFZ0SUZCRVJpQlFiSFZuYVc0aUxDSmtaWE5qY21sd2RHbHZiaUk2SWxCdmNuUmhZbXhsSUVSdlkzVnRaVzUwSUVadmNtMWhkQ0lzSW0xcGJXVlVlWEJsY3lJNlczc2lkSGx3WlNJNkltRndjR3hwWTJGMGFXOXVMM2d0WjI5dloyeGxMV05vY205dFpTMXdaR1lpTENKemRXWm1hWGhsY3lJNkluQmtaaUo5WFgwc2V5SnVZVzFsSWpvaVEyaHliMjFwZFcwZ1VFUkdJRlpwWlhkbGNpSXNJbVJsYzJOeWFYQjBhVzl1SWpvaUlpd2liV2x0WlZSNWNHVnpJanBiZXlKMGVYQmxJam9pWVhCd2JHbGpZWFJwYjI0dmNHUm1JaXdpYzNWbVptbDRaWE1pT2lKd1pHWWlmVjE5WFE9PSxXeUo2YUMxRFRpSmQsMCwxLDAsMjQsMjM3NDE1OTMwLDgsMjI3MTI2NTIwLDAsMSwwLC00OTEyNzU1MjMsUjI5dloyeGxJRWx1WXk0Z1RtVjBjMk5oY0dVZ1IyVmphMjhnVjJsdU16SWdOUzR3SUNoWGFXNWtiM2R6SUU1VUlERXdMakE3SUZkcGJqWTBPeUI0TmpRcElFRndjR3hsVjJWaVMybDBMelV6Tnk0ek5pQW9TMGhVVFV3c0lHeHBhMlVnUjJWamEyOHBJRU5vY205dFpTOHhNamN1TUM0d0xqQWdVMkZtWVhKcEx6VXpOeTR6TmlBeU1EQXpNREV3TnlCTmIzcHBiR3hoLGV5SmphSEp2YldVaU9uc2lZWEJ3SWpwN0ltbHpTVzV6ZEdGc2JHVmtJanBtWVd4elpTd2lTVzV6ZEdGc2JGTjBZWFJsSWpwN0lrUkpVMEZDVEVWRUlqb2laR2x6WVdKc1pXUWlMQ0pKVGxOVVFVeE1SVVFpT2lKcGJuTjBZV3hzWldRaUxDSk9UMVJmU1U1VFZFRk1URVZFSWpvaWJtOTBYMmx1YzNSaGJHeGxaQ0o5TENKU2RXNXVhVzVuVTNSaGRHVWlPbnNpUTBGT1RrOVVYMUpWVGlJNkltTmhibTV2ZEY5eWRXNGlMQ0pTUlVGRVdWOVVUMTlTVlU0aU9pSnlaV0ZrZVY5MGIxOXlkVzRpTENKU1ZVNU9TVTVISWpvaWNuVnVibWx1WnlKOWZYMTksNjUsLTExODM0MTA3MiwxLDEsLTEsMTY5OTk1NDg4NywxNjk5OTU0ODg3LDMzNjAwNzkzMyw4; guest=true; x-hng=lang=zh-CN&domain=www.ozon.ru; __Secure-user-id=210183128; is_adult_confirmed=; is_alco_adult_confirmed=; ozonIdAuthResponseToken=eyJhbGciOiJIUzI1NiIsIm96b25pZCI6Im5vdHNlbnNpdGl2ZSIsInR5cCI6IkpXVCJ9.eyJ1c2VyX2lkIjoyMTAxODMxMjgsImlzX3JlZ2lzdHJhdGlvbiI6ZmFsc2UsInJldHVybl91cmwiOiIiLCJwYXlsb2FkIjpudWxsLCJleHAiOjE3NTI1NDczNzksImlhdCI6MTc1MjU0NzM2OSwiaXNzIjoib3pvbmlkIn0.DX0nfNf9rcuPdt9lzE5fn1en5yqiD7Aw1vqpRt13OiU; abt_data=7.iE3bYFZ2m7yMc8mn5Rm8V9pI_ELHBH8eHNcM1w0kxMd8-HXap37uTEk6E_nAsmdWsO5pYQKhwamysCHZexl_YPPpWOWk7wgfKSuP8pTEdlDlXwLOy-sokLKLOdHyTFxcxNx5yRfpmNqFoQP8D5KccoiDh5U5kU8x7rJDLpqixSah6TFKKYsTiZrokn5Tb5aJu5lMAZBOkhr7CkYTFd_4j9wtALKnFM-oZGxCX0qTgUP5kIf9MfDSGI0U0pZ6igW6aSGirFb5ZVNmCV2D4NImCGn00K_Sn8ZX0vR6krWW1cixLrCnKp0rO7JJEFi9c7-4FL54ZvaJ-tKg8ALwlPmRIr-aI156iSlHQU6lULo8oKmBL13eLI9d_d8fp50BJe7oA6PNylYhI5DdV81WGI2EFnZtR3sVYF8O4qrS7gXwsMCoku51prrYZG1pLJJiD8HfAPTnFLvxpURZ7x1-lsAinljoh8_N1oVP89PVPJjd-tKQyRiUnleAnwaeF9kQmGvIMqYSYi506QWf-KHqfdEnEA; __Secure-ETC=90ef0679199c55cdec8592a7e480c3c5; __Secure-access-token=8.210183128.eH-AgbboQkWnqX9a7XeDqQ.92.Aa5lCeHJOXOoA90AsaTtX6OsppswLMDuxLGNHW98_K79BKfyceiX8mpea_qxY5qoaYUOO7_5hsQ6ndRa9tCxUzOCvERd7f056ZaoOw8W9lnUgQn_q5TX-7X2WK8ejP7OZg.20250715024249.20250715073104.M4j1mEB0PD4T55SZ4iQqCG6lZytjYmwG50Q28F-3q_s.1a7857bd2a7e95575; __Secure-refresh-token=8.210183128.eH-AgbboQkWnqX9a7XeDqQ.92.Aa5lCeHJOXOoA90AsaTtX6OsppswLMDuxLGNHW98_K79BKfyceiX8mpea_qxY5qoaYUOO7_5hsQ6ndRa9tCxUzOCvERd7f056ZaoOw8W9lnUgQn_q5TX-7X2WK8ejP7OZg.20250715024249.20250715073104.Ip-CdBv1mEGilz15LZCTwFmfoxEL8RrayFNa90r_XLI.197047eb6a54a7041; is_cookies_accepted=1; token=eyJhbGciOiJIUzUxMiJ9.eyJsb2dpbl91c2VyX2tleSI6IjJiMDM5ODBiLWYzZDQtNDI2Ny04NTQ2LWYwZjQxOTczNTAyNiJ9.oIF-bHh7pubdNWzETutNcoc_Nu-A7zgqBIJwcHFgF0V2s-xfnZVbs_EbvyJSBYYkUjqrBlJP_1qBl3vB1mQ_ow"); + + Spider.create(new PageProcessor() { + @Override + public void process(Page page) { + List imgs = page.getHtml() + .css("div.ip7_24.p7i_24 img.i7p_24.b95_3_1-a", "src").all(); + page.putField("imageUrls", imgs); + } + + @Override + public Site getSite() { + return site; + } + }).addUrl(url) + .addPipeline((resultItems, task) -> { + List imgs = resultItems.get("imageUrls"); + if (imgs != null) { + imageUrls.addAll(imgs); + } + }) + .thread(1) + .run(); + + return AjaxResult.success(imageUrls); + } +} diff --git a/ruoyi-admin/src/main/java/com/ruoyi/web/controller/tool/RakutenController.java b/ruoyi-admin/src/main/java/com/ruoyi/web/controller/tool/RakutenController.java new file mode 100644 index 00000000..86149b67 --- /dev/null +++ b/ruoyi-admin/src/main/java/com/ruoyi/web/controller/tool/RakutenController.java @@ -0,0 +1,189 @@ +package com.ruoyi.web.controller.tool; + +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestParam; +import com.ruoyi.common.core.controller.BaseController; +import com.ruoyi.common.core.domain.AjaxResult; +import com.ruoyi.web.util.WebMagicProxyUtil; + +import us.codecraft.webmagic.Page; +import us.codecraft.webmagic.Site; +import us.codecraft.webmagic.Spider; +import us.codecraft.webmagic.processor.PageProcessor; +import us.codecraft.webmagic.downloader.HttpClientDownloader; +import us.codecraft.webmagic.proxy.Proxy; +import us.codecraft.webmagic.proxy.SimpleProxyProvider; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.net.InetSocketAddress; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * 乐天市场控制器 + * + * @author ruoyi + * @date 2025-07-15 + */ +@RestController +@RequestMapping("/prod/rakuten") +public class RakutenController extends BaseController +{ + private static final Logger logger = LoggerFactory.getLogger(RakutenController.class); + + private final Random random = new Random(); + + private static final String DEFAULT_PROXY_HOST = "127.0.0.1"; + private static final String DEFAULT_PROXY_PORT = "7890"; + + @GetMapping("/scrapeProducts") + public AjaxResult scrapeProducts() + { + String url = "https://ranking.rakuten.co.jp/search?stx=GBAmarket&smd=0&prl=&pru=&rvf=&arf=&vmd=0&ptn=1&srt=1&sgid="; + + List> products = new ArrayList<>(); + + Site site = Site.me() + .setRetryTimes(3) + .setTimeOut(10000) + .setSleepTime(1000 + random.nextInt(2000)); + + site.addHeader("cookie", "_ra=1750472997398|0ff0eb32-5d9f-4c27-a7ca-c9ff1149e90b; Rp=779873a7b6c0e87edcf6f39cf2368561928309c8; rcx=136377ca-334f-4305-b45e-ad43e9538d2e; rcxGlobal=136377ca-334f-4305-b45e-ad43e9538d2e; _cc_id=5499b2bb8cabd8f467d3c872aaaffa96; panoramaId_expiry=1752628971187; panoramaId=541b9862d3ebc6afde47204c6b41a9fb927a29cfaf47c94f7efc477bd6e08018; panoramaIdType=panoDevice; x-hng=lang=zh-CN&domain=ranking.rakuten.co.jp; rat_v=b15ac773a5d64baf8dca92de5e06876098f84f76; Re=22.9.18.0.0.101839.1-22.9.18.0.0.101839.1; ak_bmsc=97272C637D70F43BF1F202AB32536B36~000000000000000000000000000000~YAAQGPAgF+Lm18yXAQAAsL0WDRzGbOBBlfAvGsV2IKdcTjZCiC3Uq+Y/Dmh6JHW6gn4q5X/8zVgCj7wEV/E+uAU4gX+3Yp/I1QvPmBxAOEgkawuEiktjCym641i25ZuBJ2CmpgqwGujbe7rDVR3pBCSzoR6AMAr4/JPfI8UjhcZ8QHDm/2RhnYM50NUBTz2Bb96oYmHmhiFxFrOrFbHbMJoEmXmOAXINh35/U6uR/q3iJG9psPoQMWrTHt36xGIzrD1WmEoOpDIgxGKtJdqWiI2HWYouhUrUm2LfJB3xqqKsDlEbzJKATdP1aZ8BaxZXLKgFuucr8fAYV2JXmQVlgAAAWMoi1BvgX31EVIo/HrbtXwqUWV+axZTgbRIaUzX3MUU2lLT9YqSrxE0U3RZeqnyTQLnNEtUhzzPb5ZtHJevwoHnihvv1/A2NrouNJDp2FVnORkSqjovEs9l/Z5CL/RI=; cto_bundle=Ho3NGF9iMU05dDg3TFBta25zOWUlMkJrbVVPSXRieXhvQkE2OHJwZ2Z2a3JoMElUak9LVEVYZHB5cXQ2N0pHTXpudHlHRXpodE9CUENETnRMdzhkQ1VjZU96UHhXVGpYcSUyQnZyd21ueDIwWTR2MWlPbGM3WG94ZHpqQW43R0NqViUyRjJ4NFEzNUN4OVFjRCUyQkFYVG1VTTdiQmMlMkZPY0dnJTNEJTNE; _fbp=fb.2.1752566251478.873208328842983622; __lt__cid.3df24f5b=f5a060ab-0930-48f9-9658-8d1e546fcc57; _gcl_au=1.1.1606205613.1752566252; _tt_enable_cookie=1; _ttp=01K06HDGJ7PVRWNX5APHE62MH7_.tt.2; ttcsid_COAFPAJC77U4F0RAECNG=1752566252155::HgNyOTPjyMcNbhYtz-AB.1.1752566252155; ttcsid=1752566252106::mH9Grdckk_3SH1Bp6UpS.1.1752566252155; _uetsid=5cd4d200615111f0adebf3cbc093b02c; _uetvid=5cd50bd0615111f099aad7893516afe1; ttcsid_COAECTBC77U6F5DVOFS0=1752566252105::NC3L-b3aoP3YEuOGO4Qy.1.1752566253162; __gads=ID=8405c2d7614905e4:T=1752542570:RT=1752567475:S=ALNI_MaVrnkQdqffpLsBqCP5rcOOwIHewA; __gpi=UID=00001161dccd98cd:T=1752542570:RT=1752567475:S=ALNI_MZJ2ECy0VeTmUs9-7zNFppf4p7hOA; __eoi=ID=203c5e82c3cfee8a:T=1752542570:RT=1752567475:S=AA-AfjbN1_VcxivizXN3BbwEd-hI; FCNEC=%5B%5B%22AKsRol-HQYh5qu3EpcFJpGA81sC9pMn6YPm8F2rJtAybtg_7Cxn5XyIO04J9m_6HYBryz3D_sQi4P7sqtyVUhgj5ErkOXscKPBp34F1WjlPK6qU1uMHdxkKlXh57F7w4PMg2NVNniULz_4s12zYbVlemYykDcysxIA%3D%3D%22%5D%5D; Rb=1s80a73M01&omnir34; Rz=A8_cJHchUvQYPegyHwUBQ1PclSzsfP0rVH_Yt2voU0RIPEm8XdGVcjZetzEPlSpPcGr2h32POT39MAYXZcpjwoMSYDiGKJSIKsyWKl1d3Ley; Rq=M01.1&687617b7; Ra=A78vLV4ixp9ENQ1IvxXsZGSpfP-4acHSGn2AUu4KQz2jP7zLjLOPOjOkPdMeqbFaieOrKh3nQNfqaalxrq86sdubCJcxt26r3k-IdgcqiMMowQ~~"); + site.addHeader("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36"); + site.addHeader("accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7"); + site.addHeader("accept-encoding", "gzip, deflate, br, zstd"); + site.addHeader("accept-language", "zh-CN,zh;q=0.9"); + site.addHeader("cache-control", "max-age=0"); + site.addHeader("priority", "u=0, i"); + site.addHeader("referer", "https://login.account.rakuten.com/"); + site.addHeader("sec-ch-ua", "\"Not)A;Brand\";v=\"99\", \"Microsoft Edge\";v=\"127\", \"Chromium\";v=\"127\""); + site.addHeader("sec-ch-ua-mobile", "?0"); + site.addHeader("sec-ch-ua-platform", "\"Windows\""); + site.addHeader("sec-fetch-dest", "document"); + site.addHeader("sec-fetch-mode", "navigate"); + site.addHeader("sec-fetch-site", "cross-site"); + site.addHeader("sec-fetch-user", "?1"); + site.addHeader("upgrade-insecure-requests", "1"); + + System.setProperty("http.proxyHost", DEFAULT_PROXY_HOST); + System.setProperty("http.proxyPort", DEFAULT_PROXY_PORT); + System.setProperty("https.proxyHost", DEFAULT_PROXY_HOST); + System.setProperty("https.proxyPort", DEFAULT_PROXY_PORT); + + HttpClientDownloader httpClientDownloader = new HttpClientDownloader(); + httpClientDownloader.setProxyProvider(SimpleProxyProvider.from( + new Proxy(DEFAULT_PROXY_HOST, Integer.parseInt(DEFAULT_PROXY_PORT)) + )); + + + Spider.create(new PageProcessor() { + @Override + public void process(Page page) { + // 使用指定的HTML结构提取所有店铺信息 + List shopElements = page.getHtml().css("div.srhPic div.rnkRanking_bigImageBox").all(); + + for (String element : shopElements) { + Map product = new HashMap<>(); + + String productUrl = page.getHtml().xpath(element + "/a/@href").toString(); + if (productUrl != null) { + String[] parts = productUrl.split("/"); + if (parts.length > 3) { + product.put("shopName", parts[3]); // 从URL提取店铺名称 + } + } + + String imageUrl = page.getHtml().xpath(element + "/a/img/@src").toString(); + if (imageUrl != null) { + product.put("imageUrl", imageUrl); + } + + if (productUrl != null) { + product.put("productUrl", productUrl); + } + + if (!product.isEmpty()) { + products.add(product); + } + } + + if (products.isEmpty()) { + + List urls = page.getHtml().xpath("//div[@class='srhPic']//div[@class='rnkRanking_bigImageBox']/a/@href").all(); + List images = page.getHtml().xpath("//div[@class='srhPic']//div[@class='rnkRanking_bigImageBox']/a/img/@src").all(); + + for (int i = 0; i < Math.min(urls.size(), images.size()); i++) { + Map product = new HashMap<>(); + String productUrl = urls.get(i); + String imageUrl = images.get(i); + + String[] parts = productUrl.split("/"); + if (parts.length > 3) { + product.put("shopName", parts[3]); + } + + product.put("imageUrl", imageUrl); + product.put("productUrl", productUrl); + + products.add(product); + } + } + + if (products.isEmpty()) { + logger.info("未找到商品数据，页面内容: {}", page.getHtml().toString()); + logger.info("HTTP状态码: {}", page.getStatusCode()); + logger.info("请求URL: {}", page.getUrl().toString()); + } + } + + @Override + public Site getSite() { + return site; + } + }) + .addUrl(url) + .setDownloader(httpClientDownloader) + .thread(1) + .run(); + + clearSystemProxy(); + + return AjaxResult.success(products); + } + + /** + * 获取所有可用代理节点 + */ + @GetMapping("/proxies") + public AjaxResult getProxies() { + List> proxies = new ArrayList<>(); + Map defaultProxy = new HashMap<>(); + defaultProxy.put("name", "默认代理"); + defaultProxy.put("server", DEFAULT_PROXY_HOST); + defaultProxy.put("port", DEFAULT_PROXY_PORT); + defaultProxy.put("type", "http"); + proxies.add(defaultProxy); + + return AjaxResult.success(proxies); + } + + /** + * 清除系统代理设置 + */ + private void clearSystemProxy() { + System.clearProperty("http.proxyHost"); + System.clearProperty("http.proxyPort"); + System.clearProperty("https.proxyHost"); + System.clearProperty("https.proxyPort"); + System.clearProperty("socksProxyHost"); + System.clearProperty("socksProxyPort"); + } +} diff --git a/ruoyi-admin/src/main/java/com/ruoyi/web/controller/tool/WebMagicController.java b/ruoyi-admin/src/main/java/com/ruoyi/web/controller/tool/WebMagicController.java index cfb256f1..a7de62cd 100644 --- a/ruoyi-admin/src/main/java/com/ruoyi/web/controller/tool/WebMagicController.java +++ b/ruoyi-admin/src/main/java/com/ruoyi/web/controller/tool/WebMagicController.java @@ -398,7 +398,7 @@ public class WebMagicController extends BaseController implements PageProcessor String price = (String) resultMap.get("price"); String resultAsin = (String) resultMap.get("asin"); - + if (resultAsin == null || resultAsin.isEmpty() || price == null || price.isEmpty()) { logger.info("ASIN: {} 首次爬取结果不完整，等待后重试", asin); Thread.sleep(1000 + random.nextInt(20000)); @@ -451,7 +451,6 @@ public class WebMagicController extends BaseController implements PageProcessor break; } int waitSeconds = 30 + random.nextInt(21); - logger.info("第{}次重试，等待{}秒...", retryRound + 1, waitSeconds); Thread.sleep(waitSeconds * 1000); List currentFailedAsins = new ArrayList<>(failedAsins); @@ -466,8 +465,6 @@ public class WebMagicController extends BaseController implements PageProcessor .addUrl(url) .setDownloader(getProxyDownloader()) .thread(1); - - // 记录当前活动的爬虫 synchronized (spiderLock) { activeSpider = spider; } @@ -501,20 +498,11 @@ public class WebMagicController extends BaseController implements PageProcessor } logger.info("第{}次重试完成，还有{}个ASIN未成功获取数据", retryRound + 1, failedAsins.size()); } - - if (failedAsins.isEmpty()) { - logger.info("所有ASIN均成功获取数据"); - } else { - logger.warn("经过3次重试，仍有{}个ASIN未能成功获取数据", failedAsins.size()); - } } - // 清除爬虫引用 synchronized (spiderLock) { activeSpider = null; } - - logger.info("批量爬取完成，结果数量: {}", resultList.size()); deferredResult.setResult(ResponseEntity.ok(R.ok(resultList))); } catch (Exception e) { clearSystemProxy(); diff --git a/ruoyi-ui/src/api/prod/ozon.js b/ruoyi-ui/src/api/prod/ozon.js new file mode 100644 index 00000000..075e32f6 --- /dev/null +++ b/ruoyi-ui/src/api/prod/ozon.js @@ -0,0 +1,9 @@ +import request from '@/utils/request' + +// 爬取图片 +export function scrapeImages() { + return request({ + url: '/prod/ozon/scrapeImages', + method: 'get' + }) +} diff --git a/ruoyi-ui/src/api/prod/rakuten.js b/ruoyi-ui/src/api/prod/rakuten.js new file mode 100644 index 00000000..f7adf9f0 --- /dev/null +++ b/ruoyi-ui/src/api/prod/rakuten.js @@ -0,0 +1,9 @@ +import request from '@/utils/request' + +// 从乐天市场抓取商品数据 +export function scrapeRakutenProducts() { + return request({ + url: '/prod/rakuten/scrapeProducts', + method: 'get' + }) +} diff --git a/ruoyi-ui/src/views/prod/ozon/index.vue b/ruoyi-ui/src/views/prod/ozon/index.vue new file mode 100644 index 00000000..27b0742e --- /dev/null +++ b/ruoyi-ui/src/views/prod/ozon/index.vue @@ -0,0 +1,85 @@ + + + + + diff --git a/ruoyi-ui/src/views/prod/rakuten/index.vue b/ruoyi-ui/src/views/prod/rakuten/index.vue new file mode 100644 index 00000000..aec57450 --- /dev/null +++ b/ruoyi-ui/src/views/prod/rakuten/index.vue @@ -0,0 +1,110 @@ + + + + +