增加爬虫

jfb-recruit/pom.xml

@@ -199,6 +199,12 @@
             <version>4.40.461.ALL</version>
         </dependency>
 
+        <!-- Jsoup依赖：版本选1.17.2（最新稳定版），兼容JDK8+ -->
+        <dependency>
+            <groupId>org.jsoup</groupId>
+            <artifactId>jsoup</artifactId>
+            <version>1.17.2</version>
+        </dependency>
     </dependencies>
 
     <build>

jfb-recruit/src/main/java/com/jfb/recruit/controller/api/ApiJsonpController.java

+package com.jfb.recruit.controller.api;
+
+import base.controller.BaseController;
+import base.result.BaseResult;
+import org.jsoup.Jsoup;
+import org.jsoup.nodes.Document;
+import org.jsoup.nodes.Element;
+import org.jsoup.select.Elements;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+
+import javax.servlet.http.HttpServletRequest;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+
+
+/**
+ * @author wangmenglong
+ * @description: 爬虫
+ * @date 2023/12/11 11:35
+ */
+@RestController
+@RequestMapping("/public/jsonp")
+public class ApiJsonpController extends BaseController {
+
+    private static final List<String> USER_AGENT_POOL = new ArrayList<>();
+    static {
+        USER_AGENT_POOL.add("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36");
+        USER_AGENT_POOL.add("Mozilla/5.0 (Macintosh; Intel Mac OS X 14_0) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Safari/605.1.15");
+        USER_AGENT_POOL.add("Mozilla/5.0 (Linux; Android 13; SM-G998B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Mobile Safari/537.36");
+    }
+
+    // 2. 随机获取User-Agent
+    private static String getRandomUserAgent() {
+        Random random = new Random();
+        return USER_AGENT_POOL.get(random.nextInt(USER_AGENT_POOL.size()));
+    }
+
+    /**
+     * @description: 获取openId
+     * @author: wangmenglong
+     * @date; 2023/12/7 10:07
+     * @param: [req]
+     * @return: com.github.pagehelper.PageInfo
+     **/
+    @GetMapping("/run")
+    public BaseResult run(HttpServletRequest req){
+
+        // 1. 目标爬取网址（静态页面，无反爬）
+        String targetUrl = req.getParameter("url");
+
+        try {
+            // 2. 模拟浏览器发送请求（设置User-Agent避免被识别为爬虫）
+            Document document = Jsoup.connect(targetUrl)
+                    .userAgent(getRandomUserAgent()) // 随机User-Agent
+                    .referrer("https://www.baidu.com") // 模拟从百度跳转（Referer）
+                    .header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8") // 接受的内容类型
+                    .header("Accept-Language", "zh-CN,zh;q=0.9,en;q=0.8") // 语言偏好
+                    .header("Cache-Control", "max-age=0") // 缓存控制
+                    .timeout(10000) // 超时时间10秒
+                    .get(); // 发送GET请求
+
+            // 3. 解析HTML：通过CSS选择器提取目标数据（章节标题）
+            // 先在网页F12查看元素，找到标题对应的CSS选择器
+            Elements chapterTitles = document.select("div.leftCon + ul li a");
+
+            // 4. 遍历并输出爬取到的数据
+            System.out.println("Java教程章节列表：");
+            for (Element title : chapterTitles) {
+                String chapterName = title.text(); // 提取标题文本
+                System.out.println("- " + chapterName);
+            }
+            return BaseResult.success();
+        } catch (IOException e) {
+            // 异常处理：捕获网络请求/解析失败的情况
+            System.err.println("爬虫执行失败：" + e.getMessage());
+            e.printStackTrace();
+        }
+        return BaseResult.success();
+
+    }
+
+}