From 5437bf824a6e7085e5b89f80599b75f41162561d Mon Sep 17 00:00:00 2001 From: kunfei Date: Sun, 23 Jun 2019 11:54:07 +0800 Subject: [PATCH] up --- app/build.gradle | 1 + .../app/model/analyzeRule/AnalyzeByJSoup.kt | 435 ++++++++++++++++++ .../app/model/analyzeRule/AnalyzeByXPath.kt | 160 +++++++ .../app/model/analyzeRule/analyzeJsoup.kt | 2 - .../io/legado/app/utils/StringExtensions.kt | 5 +- 5 files changed, 600 insertions(+), 3 deletions(-) create mode 100644 app/src/main/java/io/legado/app/model/analyzeRule/AnalyzeByJSoup.kt create mode 100644 app/src/main/java/io/legado/app/model/analyzeRule/AnalyzeByXPath.kt delete mode 100644 app/src/main/java/io/legado/app/model/analyzeRule/analyzeJsoup.kt diff --git a/app/build.gradle b/app/build.gradle index 819a06b2e..97efb735b 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -115,6 +115,7 @@ dependencies { //规则相关 implementation 'com.google.code.gson:gson:2.8.5' implementation 'org.jsoup:jsoup:1.12.1' + implementation 'cn.wanghaomiao:JsoupXpath:2.3.2' // implementation 'org.apache.commons:commons-text:1.6' // Default json parser for jsonpath implementation 'net.minidev:json-smart:2.3' diff --git a/app/src/main/java/io/legado/app/model/analyzeRule/AnalyzeByJSoup.kt b/app/src/main/java/io/legado/app/model/analyzeRule/AnalyzeByJSoup.kt new file mode 100644 index 000000000..119ae12ce --- /dev/null +++ b/app/src/main/java/io/legado/app/model/analyzeRule/AnalyzeByJSoup.kt @@ -0,0 +1,435 @@ +package io.legado.app.model.analyzeRule + +import android.text.TextUtils.isEmpty +import android.text.TextUtils.join +import io.legado.app.utils.startWithIgnoreCase +import org.jsoup.Jsoup +import org.jsoup.nodes.Element +import org.jsoup.select.Collector +import org.jsoup.select.Elements +import org.jsoup.select.Evaluator +import org.seimicrawler.xpath.JXNode +import java.util.* + +/** + * Created by GKF on 2018/1/25. + * 书源规则解析 + */ + +class AnalyzeByJSoup { + private var element: Element? = null + + fun parse(doc: Any): AnalyzeByJSoup { + if (doc is Element) { + element = doc + } else if (doc is JXNode) { + if (doc.isElement) { + element = doc.asElement() + } else { + element = Jsoup.parse(doc.value().toString()) + } + } else { + element = Jsoup.parse(doc.toString()) + } + return this + } + + /** + * 获取列表 + */ + internal fun getElements(rule: String): Elements { + return getElements(element, rule) + } + + /** + * 合并内容列表,得到内容 + */ + internal fun getString(ruleStr: String): String? { + if (isEmpty(ruleStr)) { + return null + } + val textS = getStringList(ruleStr) + return if (textS.isEmpty()) { + null + } else join("\n", textS).trim { it <= ' ' } + } + + /** + * 获取一个字符串 + */ + internal fun getString0(ruleStr: String): String { + val urlList = getStringList(ruleStr) + return if (urlList.isNotEmpty()) { + urlList[0] + } else "" + } + + /** + * 获取所有内容列表 + */ + internal fun getStringList(ruleStr: String): List { + val textS = ArrayList() + if (isEmpty(ruleStr)) { + return textS + } + //拆分规则 + val sourceRule = SourceRule(ruleStr) + if (isEmpty(sourceRule.elementsRule)) { + textS.add(element!!.data()) + } else { + val elementsType: String + val ruleStrS: Array + when { + sourceRule.elementsRule.contains("&") -> { + elementsType = "&" + ruleStrS = + sourceRule.elementsRule.split("&+".toRegex()).dropLastWhile { it.isEmpty() }.toTypedArray() + } + sourceRule.elementsRule.contains("%%") -> { + elementsType = "%" + ruleStrS = + sourceRule.elementsRule.split("%%".toRegex()).dropLastWhile { it.isEmpty() }.toTypedArray() + } + else -> { + elementsType = "|" + ruleStrS = if (sourceRule.isCss) { + sourceRule.elementsRule.split("\\|\\|".toRegex()).dropLastWhile { it.isEmpty() }.toTypedArray() + } else { + sourceRule.elementsRule.split("\\|+".toRegex()).dropLastWhile { it.isEmpty() }.toTypedArray() + } + } + } + val results = ArrayList>() + for (ruleStrX in ruleStrS) { + val temp: List? + temp = if (sourceRule.isCss) { + val lastIndex = ruleStrX.lastIndexOf('@') + getResultLast( + element!!.select(ruleStrX.substring(0, lastIndex)), + ruleStrX.substring(lastIndex + 1) + ) + } else { + getResultList(ruleStrX) + } + if (temp != null && temp.isNotEmpty()) { + results.add(temp) + if (results.isNotEmpty() && elementsType == "|") { + break + } + } + } + if (results.size > 0) { + if ("%" == elementsType) { + for (i in 0 until results[0].size) { + for (temp in results) { + if (i < temp.size) { + textS.add(temp[i]) + } + } + } + } else { + for (temp in results) { + textS.addAll(temp) + } + } + } + } + if (!isEmpty(sourceRule.replaceRegex)) { + val tempList = ArrayList(textS) + textS.clear() + for (text in tempList) { + val tmpText = text.replace(sourceRule.replaceRegex.toRegex(), sourceRule.replacement) + if (tmpText.isNotEmpty()) { + textS.add(tmpText) + } + } + } + return textS + } + + /** + * 获取Elements + */ + private fun getElements(temp: Element?, rule: String): Elements { + val elements = Elements() + if (temp == null || isEmpty(rule)) { + return elements + } + val sourceRule = SourceRule(rule) + val elementsType: String + val ruleStrS: Array + when { + sourceRule.elementsRule.contains("&") -> { + elementsType = "&" + ruleStrS = sourceRule.elementsRule.split("&+".toRegex()).dropLastWhile { it.isEmpty() }.toTypedArray() + } + sourceRule.elementsRule.contains("%") -> { + elementsType = "%" + ruleStrS = sourceRule.elementsRule.split("%+".toRegex()).dropLastWhile { it.isEmpty() }.toTypedArray() + } + else -> { + elementsType = "|" + ruleStrS = if (sourceRule.isCss) { + sourceRule.elementsRule.split("\\|\\|".toRegex()).dropLastWhile { it.isEmpty() }.toTypedArray() + } else { + sourceRule.elementsRule.split("\\|+".toRegex()).dropLastWhile { it.isEmpty() }.toTypedArray() + } + } + } + val elementsList = ArrayList() + if (sourceRule.isCss) { + for (ruleStr in ruleStrS) { + val tempS = temp.select(ruleStr) + elementsList.add(tempS) + if (tempS.size > 0 && elementsType == "|") { + break + } + } + } else { + for (ruleStr in ruleStrS) { + val tempS = getElementsSingle(temp, ruleStr) + elementsList.add(tempS) + if (tempS.size > 0 && elementsType == "|") { + break + } + } + } + if (elementsList.size > 0) { + if ("%" == elementsType) { + for (i in 0 until elementsList[0].size) { + for (es in elementsList) { + if (i < es.size) { + elements.add(es[i]) + } + } + } + } else { + for (es in elementsList) { + elements.addAll(es) + } + } + } + return elements + } + + private fun filterElements(elements: Elements, rules: Array?): Elements { + if (rules == null || rules.size < 2) return elements + val selectedEls = Elements() + for (ele in elements) { + var isOk = false + when (rules[0]) { + "class" -> isOk = ele.getElementsByClass(rules[1]).size > 0 + "id" -> isOk = ele.getElementById(rules[1]) != null + "tag" -> isOk = ele.getElementsByTag(rules[1]).size > 0 + "text" -> isOk = ele.getElementsContainingOwnText(rules[1]).size > 0 + } + if (isOk) { + selectedEls.add(ele) + } + } + return selectedEls + } + + /** + * 获取Elements按照一个规则 + */ + private fun getElementsSingle(temp: Element, rule: String): Elements { + val elements = Elements() + try { + val rs = rule.trim { it <= ' ' }.split("@".toRegex()).dropLastWhile { it.isEmpty() }.toTypedArray() + if (rs.size > 1) { + elements.add(temp) + for (rl in rs) { + val es = Elements() + for (et in elements) { + es.addAll(getElements(et, rl)) + } + elements.clear() + elements.addAll(es) + } + } else { + val rulePcx = rule.split("!".toRegex()).dropLastWhile { it.isEmpty() }.toTypedArray() + val rulePc = + rulePcx[0].trim { it <= ' ' }.split(">".toRegex()).dropLastWhile { it.isEmpty() }.toTypedArray() + val rules = + rulePc[0].trim { it <= ' ' }.split("\\.".toRegex()).dropLastWhile { it.isEmpty() }.toTypedArray() + var filterRules: Array? = null + var needFilterElements = rulePc.size > 1 && !isEmpty(rulePc[1].trim { it <= ' ' }) + if (needFilterElements) { + filterRules = rulePc[1].trim { it <= ' ' }.split("\\.".toRegex()).dropLastWhile { it.isEmpty() } + .toTypedArray() + filterRules[0] = filterRules[0].trim { it <= ' ' } + val validKeys = listOf("class", "id", "tag", "text") + if (filterRules.size < 2 || !validKeys.contains(filterRules[0]) || isEmpty(filterRules[1].trim { it <= ' ' })) { + needFilterElements = false + } + filterRules[1] = filterRules[1].trim { it <= ' ' } + } + when (rules[0]) { + "children" -> { + var children = temp.children() + if (needFilterElements) + children = filterElements(children, filterRules) + elements.addAll(children) + } + "class" -> { + var elementsByClass = temp.getElementsByClass(rules[1]) + if (rules.size == 3) { + val index = Integer.parseInt(rules[2]) + if (index < 0) { + elements.add(elementsByClass[elementsByClass.size + index]) + } else { + elements.add(elementsByClass[index]) + } + } else { + if (needFilterElements) + elementsByClass = filterElements(elementsByClass, filterRules) + elements.addAll(elementsByClass) + } + } + "tag" -> { + var elementsByTag = temp.getElementsByTag(rules[1]) + if (rules.size == 3) { + val index = Integer.parseInt(rules[2]) + if (index < 0) { + elements.add(elementsByTag[elementsByTag.size + index]) + } else { + elements.add(elementsByTag[index]) + } + } else { + if (needFilterElements) + elementsByTag = filterElements(elementsByTag, filterRules) + elements.addAll(elementsByTag) + } + } + "id" -> { + var elementsById = Collector.collect(Evaluator.Id(rules[1]), temp) + if (rules.size == 3) { + val index = Integer.parseInt(rules[2]) + if (index < 0) { + elements.add(elementsById[elementsById.size + index]) + } else { + elements.add(elementsById[index]) + } + } else { + if (needFilterElements) + elementsById = filterElements(elementsById, filterRules) + elements.addAll(elementsById) + } + } + "text" -> { + var elementsByText = temp.getElementsContainingOwnText(rules[1]) + if (needFilterElements) + elementsByText = filterElements(elementsByText, filterRules) + elements.addAll(elementsByText) + } + else -> elements.addAll(temp.select(rulePcx[0])) + } + if (rulePcx.size > 1) { + val rulePcs = rulePcx[1].split(":".toRegex()).dropLastWhile { it.isEmpty() }.toTypedArray() + for (pc in rulePcs) { + val pcInt = Integer.parseInt(pc) + if (pcInt < 0 && elements.size + pcInt >= 0) { + elements[elements.size + pcInt] = null + } else if (Integer.parseInt(pc) < elements.size) { + elements[Integer.parseInt(pc)] = null + } + } + val es = Elements() + es.add(null) + elements.removeAll(es) + } + } + } catch (ignore: Exception) { + } + + return elements + } + + /** + * 获取内容列表 + */ + private fun getResultList(ruleStr: String): List? { + if (isEmpty(ruleStr)) { + return null + } + var elements = Elements() + elements.add(element) + val rules = ruleStr.split("@".toRegex()).dropLastWhile { it.isEmpty() }.toTypedArray() + for (i in 0 until rules.size - 1) { + val es = Elements() + for (elt in elements) { + es.addAll(getElementsSingle(elt, rules[i])) + } + elements.clear() + elements = es + } + return if (elements.isEmpty()) { + null + } else getResultLast(elements, rules[rules.size - 1]) + } + + /** + * 根据最后一个规则获取内容 + */ + private fun getResultLast(elements: Elements, lastRule: String): List { + val textS = ArrayList() + try { + when (lastRule) { + "text" -> for (element in elements) { + val text = element.text() + textS.add(text) + } + "textNodes" -> for (element in elements) { + val contentEs = element.textNodes() + for (i in contentEs.indices) { + val temp = contentEs[i].text().trim { it <= ' ' } + if (!isEmpty(temp)) { + textS.add(temp) + } + } + } + "ownText", "html" -> { + elements.select("script").remove() + val html = elements.html() + textS.add(html) + } + else -> for (element in elements) { + val url = element.attr(lastRule) + if (!isEmpty(url) && !textS.contains(url)) { + textS.add(url) + } + } + } + } catch (ignore: Exception) { + } + + return textS + } + + internal inner class SourceRule(ruleStr: String) { + var isCss = false + var elementsRule: String + var replaceRegex = "" + var replacement = "" + + init { + if (startWithIgnoreCase(ruleStr, "@CSS:")) { + isCss = true + elementsRule = ruleStr.substring(5).trim { it <= ' ' } + } else { + val ruleStrS: Array = + ruleStr.trim { it <= ' ' }.split("#".toRegex()).dropLastWhile { it.isEmpty() }.toTypedArray() + //分离正则表达式 + elementsRule = ruleStrS[0] + if (ruleStrS.size > 1) { + replaceRegex = ruleStrS[1] + } + if (ruleStrS.size > 2) { + replacement = ruleStrS[2] + } + } + } + } + +} diff --git a/app/src/main/java/io/legado/app/model/analyzeRule/AnalyzeByXPath.kt b/app/src/main/java/io/legado/app/model/analyzeRule/AnalyzeByXPath.kt new file mode 100644 index 000000000..eca6674c2 --- /dev/null +++ b/app/src/main/java/io/legado/app/model/analyzeRule/AnalyzeByXPath.kt @@ -0,0 +1,160 @@ +package io.legado.app.model.analyzeRule + +import android.text.TextUtils +import org.jsoup.nodes.Document +import org.seimicrawler.xpath.JXDocument +import org.seimicrawler.xpath.JXNode +import java.util.* + +class AnalyzeByXPath { + private var jxDocument: JXDocument? = null + + fun parse(doc: Any): AnalyzeByXPath { + if (doc is Document) { + jxDocument = JXDocument.create(doc) + } else { + var html = doc.toString() + // 给表格标签添加完整的框架结构,否则会丢失表格标签;html标准不允许表格标签独立在table之外 + if (html.endsWith("")) { + html = String.format("%s", html) + } + if (html.endsWith("") || html.endsWith("")) { + html = String.format("%s
", html) + } + jxDocument = JXDocument.create(html) + } + return this + } + + internal fun getElements(xPath: String): List? { + if (TextUtils.isEmpty(xPath)) { + return null + } + val jxNodes = ArrayList() + val elementsType: String + val rules: Array + if (xPath.contains("&&")) { + rules = xPath.split("&&".toRegex()).dropLastWhile { it.isEmpty() }.toTypedArray() + elementsType = "&" + } else if (xPath.contains("%%")) { + rules = xPath.split("%%".toRegex()).dropLastWhile { it.isEmpty() }.toTypedArray() + elementsType = "%" + } else { + rules = xPath.split("\\|\\|".toRegex()).dropLastWhile { it.isEmpty() }.toTypedArray() + elementsType = "|" + } + if (rules.size == 1) { + return jxDocument!!.selN(rules[0]) + } else { + val results = ArrayList>() + for (rl in rules) { + val temp = getElements(rl) + if (temp != null && temp.isNotEmpty()) { + results.add(temp) + if (temp.isNotEmpty() && elementsType == "|") { + break + } + } + } + if (results.size > 0) { + if ("%" == elementsType) { + for (i in 0 until results[0].size) { + for (temp in results) { + if (i < temp.size) { + jxNodes.add(temp[i]) + } + } + } + } else { + for (temp in results) { + jxNodes.addAll(temp) + } + } + } + } + return jxNodes + } + + internal fun getStringList(xPath: String): List { + val result = ArrayList() + val elementsType: String + val rules: Array + if (xPath.contains("&&")) { + rules = xPath.split("&&".toRegex()).dropLastWhile { it.isEmpty() }.toTypedArray() + elementsType = "&" + } else if (xPath.contains("%%")) { + rules = xPath.split("%%".toRegex()).dropLastWhile { it.isEmpty() }.toTypedArray() + elementsType = "%" + } else { + rules = xPath.split("\\|\\|".toRegex()).dropLastWhile { it.isEmpty() }.toTypedArray() + elementsType = "|" + } + if (rules.size == 1) { + val jxNodes = jxDocument!!.selN(xPath) + for (jxNode in jxNodes) { + /*if(jxNode.isString()){ + result.add(String.valueOf(jxNode)); + }*/ + result.add(jxNode.toString()) + } + return result + } else { + val results = ArrayList>() + for (rl in rules) { + val temp = getStringList(rl) + if (temp.isNotEmpty()) { + results.add(temp) + if (temp.isNotEmpty() && elementsType == "|") { + break + } + } + } + if (results.size > 0) { + if ("%" == elementsType) { + for (i in 0 until results[0].size) { + for (temp in results) { + if (i < temp.size) { + result.add(temp[i]) + } + } + } + } else { + for (temp in results) { + result.addAll(temp) + } + } + } + } + return result + } + + fun getString(rule: String): String? { + val rules: Array + val elementsType: String + if (rule.contains("&&")) { + rules = rule.split("&&".toRegex()).dropLastWhile { it.isEmpty() }.toTypedArray() + elementsType = "&" + } else { + rules = rule.split("\\|\\|".toRegex()).dropLastWhile { it.isEmpty() }.toTypedArray() + elementsType = "|" + } + if (rules.size == 1) { + /*Object object = jxDocument.selOne(rule); + result = object instanceof Element ? ((Element) object).html() : (String) object;*/ + val jxNodes = jxDocument!!.selN(rule) ?: return null + return TextUtils.join(",", jxNodes) + } else { + val sb = StringBuilder() + for (rl in rules) { + val temp = getString(rl) + if (!TextUtils.isEmpty(temp)) { + sb.append(temp) + if (elementsType == "|") { + break + } + } + } + return sb.toString() + } + } +} diff --git a/app/src/main/java/io/legado/app/model/analyzeRule/analyzeJsoup.kt b/app/src/main/java/io/legado/app/model/analyzeRule/analyzeJsoup.kt deleted file mode 100644 index c65049d4e..000000000 --- a/app/src/main/java/io/legado/app/model/analyzeRule/analyzeJsoup.kt +++ /dev/null @@ -1,2 +0,0 @@ -package io.legado.app.model.analyzeRule - diff --git a/app/src/main/java/io/legado/app/utils/StringExtensions.kt b/app/src/main/java/io/legado/app/utils/StringExtensions.kt index 92f2801de..2706e05dd 100644 --- a/app/src/main/java/io/legado/app/utils/StringExtensions.kt +++ b/app/src/main/java/io/legado/app/utils/StringExtensions.kt @@ -10,4 +10,7 @@ fun String.isAbsUrl() = this.startsWith("http://", true) fun String.splitNotBlank(delim: String) = if (!this.contains(delim)) sequenceOf(this) else this.split(delim).asSequence().map { it.trim() }.filterNot { it.isBlank() } -// fun String.unescapeJson() = StringEscapeUtils.unescapeJson(this) \ No newline at end of file +fun startWithIgnoreCase(src: String?, obj: String?): Boolean { + if (src == null || obj == null) return false + return if (obj.length > src.length) false else src.substring(0, obj.length).equals(obj, ignoreCase = true) +} \ No newline at end of file