7 changed files with 214 additions and 4 deletions
@ -0,0 +1,59 @@ |
|||
package com.zdxt.enforcementcode.common.util; |
|||
|
|||
import com.gargoylesoftware.htmlunit.BrowserVersion; |
|||
import com.gargoylesoftware.htmlunit.NicelyResynchronizingAjaxController; |
|||
import com.gargoylesoftware.htmlunit.WebClient; |
|||
import com.gargoylesoftware.htmlunit.WebClientOptions; |
|||
import com.gargoylesoftware.htmlunit.html.DomElement; |
|||
import com.gargoylesoftware.htmlunit.html.HtmlPage; |
|||
import net.sourceforge.htmlunit.corejs.javascript.EcmaError; |
|||
|
|||
import java.util.Objects; |
|||
import java.util.logging.LogManager; |
|||
|
|||
/** |
|||
* 爬虫 读取页面工具 |
|||
* @author 陈瑞 |
|||
* @date 2024-11-11 18:33 |
|||
*/ |
|||
public class CrawlerUtil { |
|||
|
|||
public static WebClient webClient; |
|||
|
|||
static { |
|||
webClient = new WebClient(BrowserVersion.CHROME); |
|||
WebClientOptions options = webClient.getOptions(); |
|||
options.setUseInsecureSSL(true); |
|||
options.setCssEnabled(false); |
|||
options.setThrowExceptionOnScriptError(false); |
|||
options.setThrowExceptionOnFailingStatusCode(false); |
|||
options.setJavaScriptEnabled(true); |
|||
options.setTimeout(10000); |
|||
webClient.setAjaxController(new NicelyResynchronizingAjaxController()); |
|||
LogManager.getLogManager().reset(); |
|||
} |
|||
|
|||
|
|||
/** |
|||
* 爬取页面固定数据 |
|||
* @param url |
|||
* @return |
|||
* @throws Exception |
|||
*/ |
|||
public static String getCreditCode(String url) throws Exception { |
|||
HtmlPage htmlPage = null; |
|||
try { |
|||
htmlPage = webClient.getPage(url); |
|||
} catch (EcmaError e) { |
|||
} |
|||
if (Objects.nonNull(htmlPage)) { |
|||
DomElement codeElement = htmlPage.getElementById("lblEntSCCode"); |
|||
if (codeElement != null) { |
|||
// HtmlUnit 2.55+ 已移除 asText(),改用 asNormalizedText()
|
|||
String code = codeElement.asNormalizedText(); |
|||
return code; |
|||
} |
|||
} |
|||
return null; |
|||
} |
|||
} |
|||
Loading…
Reference in new issue