Compare commits
2 Commits
71b0eb0ed9
...
5be250d8b8
| Author | SHA1 | Date |
|---|---|---|
|
|
5be250d8b8 | 3 months ago |
|
|
03b3255174 | 3 months ago |
7 changed files with 214 additions and 4 deletions
@ -0,0 +1,59 @@ |
|||||
|
package com.zdxt.enforcementcode.common.util; |
||||
|
|
||||
|
import com.gargoylesoftware.htmlunit.BrowserVersion; |
||||
|
import com.gargoylesoftware.htmlunit.NicelyResynchronizingAjaxController; |
||||
|
import com.gargoylesoftware.htmlunit.WebClient; |
||||
|
import com.gargoylesoftware.htmlunit.WebClientOptions; |
||||
|
import com.gargoylesoftware.htmlunit.html.DomElement; |
||||
|
import com.gargoylesoftware.htmlunit.html.HtmlPage; |
||||
|
import net.sourceforge.htmlunit.corejs.javascript.EcmaError; |
||||
|
|
||||
|
import java.util.Objects; |
||||
|
import java.util.logging.LogManager; |
||||
|
|
||||
|
/** |
||||
|
* 爬虫 读取页面工具 |
||||
|
* @author 陈瑞 |
||||
|
* @date 2024-11-11 18:33 |
||||
|
*/ |
||||
|
public class CrawlerUtil { |
||||
|
|
||||
|
public static WebClient webClient; |
||||
|
|
||||
|
static { |
||||
|
webClient = new WebClient(BrowserVersion.CHROME); |
||||
|
WebClientOptions options = webClient.getOptions(); |
||||
|
options.setUseInsecureSSL(true); |
||||
|
options.setCssEnabled(false); |
||||
|
options.setThrowExceptionOnScriptError(false); |
||||
|
options.setThrowExceptionOnFailingStatusCode(false); |
||||
|
options.setJavaScriptEnabled(true); |
||||
|
options.setTimeout(10000); |
||||
|
webClient.setAjaxController(new NicelyResynchronizingAjaxController()); |
||||
|
LogManager.getLogManager().reset(); |
||||
|
} |
||||
|
|
||||
|
|
||||
|
/** |
||||
|
* 爬取页面固定数据 |
||||
|
* @param url |
||||
|
* @return |
||||
|
* @throws Exception |
||||
|
*/ |
||||
|
public static String getCreditCode(String url) throws Exception { |
||||
|
HtmlPage htmlPage = null; |
||||
|
try { |
||||
|
htmlPage = webClient.getPage(url); |
||||
|
} catch (EcmaError e) { |
||||
|
} |
||||
|
if (Objects.nonNull(htmlPage)) { |
||||
|
DomElement codeElement = htmlPage.getElementById("lblEntSCCode"); |
||||
|
if (codeElement != null) { |
||||
|
// HtmlUnit 2.55+ 已移除 asText(),改用 asNormalizedText()
|
||||
|
String code = codeElement.asNormalizedText(); |
||||
|
return code; |
||||
|
} |
||||
|
} |
||||
|
return null; |
||||
|
} |
||||
|
} |
||||
Loading…
Reference in new issue