From d0f87aee76612c49ce5adc94c51ac5458ccbc0c3 Mon Sep 17 00:00:00 2001 From: bushixuanqi <57338301+bushixuanqi@users.noreply.github.com> Date: Sun, 4 Jul 2021 22:17:15 +0800 Subject: [PATCH 1/3] Update AnalyzeByJSoup.kt MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 改进筛选器起始字符位置查找规则，改为逐个比对，先匹配到“(”就拉取“()”平衡，先匹配到“[”就拉取“[]”平衡，避免()和[]嵌套时出现边缘情况。 跳转筛选器匹配优先顺序，从“(”优先改为“[”优先。 --- .../app/model/analyzeRule/AnalyzeByJSoup.kt | 773 +++++++++--------- 1 file changed, 401 insertions(+), 372 deletions(-) diff --git a/app/src/main/java/io/legado/app/model/analyzeRule/AnalyzeByJSoup.kt b/app/src/main/java/io/legado/app/model/analyzeRule/AnalyzeByJSoup.kt index 3bb2e4e33..dfc5cae23 100644 --- a/app/src/main/java/io/legado/app/model/analyzeRule/AnalyzeByJSoup.kt +++ b/app/src/main/java/io/legado/app/model/analyzeRule/AnalyzeByJSoup.kt @@ -1,496 +1,525 @@ package io.legado.app.model.analyzeRule -import android.text.TextUtils.join -import androidx.annotation.Keep -import org.jsoup.Jsoup -import org.jsoup.nodes.Element -import org.jsoup.select.Collector -import org.jsoup.select.Elements -import org.jsoup.select.Evaluator -import org.seimicrawler.xpath.JXNode -import java.util.* - -/** - * Created by GKF on 2018/1/25. - * 书源规则解析 - */ -@Keep -class AnalyzeByJSoup(doc: Any) { - companion object { - /** - * "class", "id", "tag", "text", "children" - */ - val validKeys = arrayOf("class", "id", "tag", "text", "children") +//通用的规则切分处理 - fun parse(doc: Any): Element { - return when (doc) { - is Element -> doc - is JXNode -> if (doc.isElement) doc.asElement() else Jsoup.parse(doc.toString()) - else -> Jsoup.parse(doc.toString()) - } - } +class RuleAnalyzer(data: String) { + + private var queue: String = data //被处理字符串 + private var pos = 0 //处理到的位置 + + private var start = 0 //每次处理字段的开始 + private var step:Int = 0 //分割字符的长度 + + var elementsType = "" + + //当前平衡字段 + fun currBalancedString( stepStart:Int = 1 , stepEnd:Int = 1): String { //stepStart平衡字符的起始分隔字串长度，stepEnd平衡字符的结束分隔字串长度 + return queue.substring(start+stepStart,pos-stepEnd) //当前平衡字段 + } + //将pos重置为0，方便复用 + fun reSetPos() { + pos = 0 } - private var element: Element = parse(doc) + //当前拉取字段 + fun currString(): String { + return queue.substring(start,pos) //当前拉取到的字段 + } + + //剩余字串 + private fun remainingString(): String { + start = pos + pos = queue.length + return queue.substring(start) + } /** - * 获取列表 + * pos位置回退 */ - internal fun getElements(rule: String) = getElements(element, rule) + fun back(num :Int = 0) { + if(num == 0)pos = start //回退 + else pos -= num + } /** - * 合并内容列表,得到内容 + * pos位置后移 */ - internal fun getString(ruleStr: String) = - if(ruleStr.isEmpty()) null - else getStringList(ruleStr).takeIf { it.isNotEmpty() }?.joinToString("\n") + fun advance(num :Int = 1) { + pos+=num + } /** - * 获取一个字符串 + * 是否已无剩余字符? + * @return 若剩余字串中已无字符则返回true */ - internal fun getString0(ruleStr: String) = getStringList(ruleStr).let{ if ( it.isEmpty() ) "" else it[0] } + val isEmpty: Boolean + get() = queue.length - pos == 0 //是否处理到最后 /** - * 获取所有内容列表 + * 检索并返回首字符,但pos不变 + * @return 首字符：若为空则为 0 号字符 */ - internal fun getStringList(ruleStr: String): List { - - val textS = ArrayList() - - if (ruleStr.isEmpty()) return textS - - //拆分规则 - val sourceRule = SourceRule(ruleStr) - - if (sourceRule.elementsRule.isEmpty()) { - - textS.add(element.data() ?: "") - - } else { - - val ruleAnalyzes = RuleAnalyzer(sourceRule.elementsRule) - val ruleStrS = ruleAnalyzes.splitRule("&&","||" ,"%%") - - val results = ArrayList>() - for (ruleStrX in ruleStrS) { + fun peek(): Char { //检索首字符 + return if (isEmpty) 0.toChar() else queue[pos] + } - val temp: List? = - if (sourceRule.isCss) { - val lastIndex = ruleStrX.lastIndexOf('@') - getResultLast( - element.select(ruleStrX.substring(0, lastIndex)), - ruleStrX.substring(lastIndex + 1) - ) - } else { - getResultList(ruleStrX) - } + /** + * 消耗剩余字串中一个字符。 + * @return 返回剩余字串中的下个字符。 + */ + fun consume(): Char { + return queue[pos++] + } - if (!temp.isNullOrEmpty()) { + /** + * 字串与剩余字串是否匹配，不区分大小写 + * @param seq 字符串被检查 + * @return 若下字符串匹配返回 true + */ + fun matches(seq: String): Boolean { + return queue.regionMatches(pos, seq, 0, seq.length, ignoreCase = true) + } - results.add(temp) //!temp.isNullOrEmpty()时，results.isNotEmpty()为true + /** + * 从剩余字串中拉出一个字符串，直到但不包括匹配序列，或剩余字串用完。 + * @param seq 分隔字符 **区分大小写** + * @return 是否找到相应字段。 + */ + fun consumeTo(seq: String,setStartPos:Boolean = true): Boolean { - if (ruleAnalyzes.elementsType == "||") break + if(setStartPos)start = pos //将处理到的位置设置为规则起点 + val offset = queue.indexOf(seq, pos) + return if (offset != -1) { + pos = offset + true + } else false - } - } - if (results.size > 0) { - if ("%%" == ruleAnalyzes.elementsType) { - for (i in results[0].indices) { - for (temp in results) { - if (i < temp.size) { - textS.add(temp[i]) - } - } - } - } else { - for (temp in results) { - textS.addAll(temp) - } - } - } - } - return textS } /** - * 获取Elements + * 从剩余字串中拉出一个字符串，直到但不包括匹配序列（匹配参数列表中一项即为匹配），或剩余字串用完。 + * @param seq 匹配字符串序列 + * @return 成功返回true并设置间隔，失败则直接返回fasle */ - private fun getElements(temp: Element?, rule: String): Elements { + fun consumeToAny(vararg seq:String): Boolean { - if (temp == null || rule.isEmpty()) return Elements() + start = pos - val elements = Elements() + while (!isEmpty) { - val sourceRule = SourceRule(rule) - val ruleAnalyzes = RuleAnalyzer(sourceRule.elementsRule) - val ruleStrS = ruleAnalyzes.splitRule("&&","||","%%") - - val elementsList = ArrayList() - if (sourceRule.isCss) { - for (ruleStr in ruleStrS) { - val tempS = temp.select(ruleStr) - elementsList.add(tempS) - if (tempS.size > 0 && ruleAnalyzes.elementsType == "||") { - break + for (s in seq) { + if (matches(s)) { + step = s.length //间隔数 + return true //匹配就返回 true } } - } else { - for (ruleStr in ruleStrS) { - //将原getElementsSingle函数调用的函数的部分代码内联过来，方便简化getElementsSingle函数 - - val rsRule = RuleAnalyzer(ruleStr) - - if( rsRule.peek() =='@' || rsRule.peek() < '!' ) rsRule.advance() // 修剪当前规则之前的"@"或者空白符 - - val rs = rsRule.splitRule("@") - - val el = if (rs.size > 1) { - val el = Elements() - el.add(temp) - for (rl in rs) { - val es = Elements() - for (et in el) { - es.addAll(getElements(et, rl)) - } - el.clear() - el.addAll(es) - } - el - }else getElementsSingle(temp,ruleStr) - elementsList.add(el) - if (el.size > 0 && ruleAnalyzes.elementsType == "||") { - break - } - } - } - if (elementsList.size > 0) { - if ("%%" == ruleAnalyzes.elementsType) { - for (i in 0 until elementsList[0].size) { - for (es in elementsList) { - if (i < es.size) { - elements.add(es[i]) - } - } - } - } else { - for (es in elementsList) { - elements.addAll(es) - } - } + pos++ //逐个试探 } - return elements - } - /** - * 1.支持阅读原有写法，':'分隔索引，!或.表示筛选方式，索引可为负数 - * - * 例如 tag.div.-1:10:2 或 tag.div!0:3 - * - * 2. 支持与jsonPath类似的[]索引写法 - * - * 格式形如 [it,it，。。。] 或 [!it,it，。。。] 其中[!开头表示筛选方式为排除，it为单个索引或区间。 - * - * 区间格式为 start:end 或 start:end:step，其中start为0可省略，end为-1可省略。 - * - * 索引，区间两端及间隔都支持负数 - * - * 例如 tag.div[-1, 3:-2:-10, 2] - * - * 特殊用法 tag.div[-1:0] 可在任意地方让列表反向 - * - * */ + pos = start //匹配失败，位置回退 - fun findIndexSet( rule:String ): IndexSet { - - val indexSet = IndexSet() - val rus = rule.trim{ it <= ' '} - - var len = rus.length - var curInt: Int? //当前数字 - var curMinus = false //当前数字是否为负 - val curList = mutableListOf() //当前数字区间 - var l = "" //暂存数字字符串 - - val head = rus.last() == ']' //是否为常规索引写法 - - if(head){ //常规索引写法[index...] - - len-- //跳过尾部']' - - while (len-- >= 0) { //逆向遍历,可以无前置规则 - - var rl = rus[len] - if (rl == ' ') continue //跳过空格 - - if (rl in '0'..'9') l += rl //将数值累接入临时字串中，遇到分界符才取出 - else if (rl == '-') curMinus = true - else { - - curInt = if (l.isEmpty()) null else if (curMinus) -l.toInt() else l.toInt() //当前数字 + return false + } - when (rl) { + /** + * 从剩余字串中拉出一个字符串，直到但不包括匹配序列（匹配参数列表中一项即为匹配），或剩余字串用完。 + * @param seq 匹配字符序列 + * @return 返回匹配位置 + */ + private fun findToAny(vararg seq:Char): Int { - ':' -> curList.add(curInt) //区间右端或区间间隔 + val start = pos //声明新变量记录临时起始位置，不更改类本身的起始位置 - else -> { + while (!isEmpty) { - //为保证查找顺序，区间和单个索引都添加到同一集合 - if(curList.isEmpty())indexSet.indexs.add(curInt!!) - else{ + for (s in seq) if(queue[pos] == s) return pos //匹配则返回位置 - //列表最后压入的是区间右端，若列表有两位则最先压入的是间隔 - indexSet.indexs.add( Triple(curInt, curList.last(), if(curList.size == 2) curList.first() else 1) ) + pos++ //逐个试探 - curList.clear() //重置临时列表，避免影响到下个区间的处理 + } - } + pos = start //匹配失败，位置回退 - if(rl == '!'){ - indexSet.split='!' - do{ rl = rus[--len] } while (len > 0 && rl == ' ')//跳过所有空格 - } + return -1 + } - if(rl == '[') return indexSet.apply { - beforeRule = rus.substring(0, len) - } //遇到索引边界，返回结果 + //其中js只要符合语法，就不用避开任何阅读关键字，自由发挥 + fun chompJsBalanced(f: ((Char) -> Boolean?) = { + when (it) { + '{' -> true //开始嵌套一层 + '}' -> false //闭合一层嵌套 + else -> null + } + } ): Boolean { + start = pos + var depth = 0 //嵌套深度 + var bracketsDepth = 0 //[]嵌套深度 + + var inSingleQuote = false //单引号 + var inDoubleQuote = false //双引号 + var inOtherQuote = false //js原始字串分隔字符 + var regex = false //正则 + var commit = false //单行注释 + var commits = false //多行注释 + + do { + if (isEmpty) break + var c = consume() + if (c != '\\') { //非转义字符 + if (c == '\'' && !commits && !commit && !regex && !inDoubleQuote && !inOtherQuote) inSingleQuote = !inSingleQuote //匹配具有语法功能的单引号 + else if (c == '"' && !commits && !commit && !regex && !inSingleQuote && !inOtherQuote) inDoubleQuote = !inDoubleQuote //匹配具有语法功能的双引号 + else if (c == '`' && !commits && !commit && !regex && !inSingleQuote && !inDoubleQuote) inOtherQuote = !inOtherQuote //匹配具有语法功能的'`' + else if (c == '/' && !commits && !commit && !regex && !inSingleQuote && !inDoubleQuote && !inOtherQuote) { //匹配注释或正则起点 + c = consume() + when(c){ + '/'->commit=true //匹配单行注释起点 + '*'->commits=true //匹配多行注释起点 + else ->regex=true //匹配正则起点 + } + } + else if(commits && c == '*') { //匹配多行注释终点 + c = consume() + if(c == '/')commits = false + } + else if(regex && c == '/') { //正则的终点或[]平衡 - if(rl != ',') break //非索引结构，跳出 + when (c) { + '/' -> regex = false//匹配正则终点 - } + //为了保证当open为（ 且 close 为 ）时，正则中[(]或[)]的合法性。故对[]这对在任何规则中都平衡的成对符号做匹配。 + // 注：正则里[(]、[)]、[{]、[}]都是合法的，所以只有[]必须平衡。 + '[' -> bracketsDepth++ //开始嵌套一层[] + ']' -> bracketsDepth-- //闭合一层嵌套[] } - l = "" //清空 - curMinus = false //重置 } - } - } else while (len-- >= 0) { //阅读原本写法，逆向遍历,可以无前置规则 - val rl = rus[len] - if (rl == ' ') continue //跳过空格 + if (commits || commit || regex || inSingleQuote || inDoubleQuote || inOtherQuote) continue //语法单元未匹配结束，直接进入下个循环 - if (rl in '0'..'9') l += rl //将数值累接入临时字串中，遇到分界符才取出 - else if (rl == '-') curMinus = true - else { + val fn = f(c) ?: continue + if (fn) depth++ else depth-- //嵌套或者闭合 - if(rl == '!' || rl == '.' || rl == ':') { //分隔符或起始符 + }else { //转义字符 + var next = consume() //拉出被转义字符 + if(commit && next == 'n') commit = false //匹配单行注释终点。当前为\,下个为n，表示换行 + else if (!commits && !commit && next == '\\') { + consume() //当前为\,下个为\，双重转义中"\\"表示转义字符本身，根据if条件"\\"字串不在注释中，则只能在字串或正则中 + next = consume() //拉出下个字符，因为在双重转义的字串或正则中，类似于 \\/ 这样的结构才是转义结构 + if(next == '\\')consume() //若为转义字符则继续拉出，因为双重转义中转义字符成对存在,即 \\\\ + } + } + } while (depth > 0 || bracketsDepth >0) //拉出全部符合js语法的字段 - indexSet.indexDefault.add(if (curMinus) -l.toInt() else l.toInt()) // 当前数字追加到列表 + if(depth > 0 || bracketsDepth >0) start = pos - if (rl != ':') return indexSet.apply { //rl == '!' || rl == '.' - split = rl - beforeRule = rus.substring(0, len) - } + return pos > start + } - }else break //非索引结构，跳出循环 + /** + * 在双重转义字串中拉出一个规则平衡组 + */ + fun chompRuleBalanced(open: Char = '[', close: Char = ']',f: ((Char) ->Boolean?)? = null ): Boolean { + start = pos + var depth = 0 //嵌套深度 + var otherDepth = 0 //其他对称符合嵌套深度 + + var inSingleQuote = false //单引号 + var inDoubleQuote = false //双引号 + + do { + if (isEmpty) break + val c = consume() + if (c != ESC) { //非转义字符 + if (c == '\'' && !inDoubleQuote) inSingleQuote = !inSingleQuote //匹配具有语法功能的单引号 + else if (c == '"' && !inSingleQuote) inDoubleQuote = !inDoubleQuote //匹配具有语法功能的双引号 + + if (inSingleQuote || inDoubleQuote) continue //语法单元未匹配结束，直接进入下个循环 + + if ( c == open )depth++ //开始嵌套一层 + else if ( c== close) depth-- //闭合一层嵌套 + else if(depth == 0 && f != null) { //处于默认嵌套中的非默认字符不需要平衡，仅depth为0时默认嵌套全部闭合，此字符才进行嵌套 + val fn = f(c) ?: continue + if (fn) otherDepth++ else otherDepth-- + } - l = "" //清空 - curMinus = false //重置 + }else { //转义字符 + var next = consume() //拉出被转义字符，匹配\/、\"、\'等 + if (next == ESC) { + consume() //当前为\,下个为\，双重转义中"\\"表示转义字符本身，根据语法特征当前字段在字串或正则中 + next = consume() //拉出下个字符，因为在双重转义的字串或正则中，类似于 \\/ 这样的结构才是转义结构 + if(next == ESC)consume() //若为转义字符则继续拉出，因为双重转义中转义字符成对存在,即 \\\\ + } } + } while (depth > 0 || otherDepth > 0) //拉出一个平衡字串 - } - - return indexSet.apply{ - split = ' ' - beforeRule = rus } //非索引格式 + return !(depth > 0 || otherDepth > 0) //平衡返回false，不平衡返回true } /** - * 获取Elements按照一个规则 + * 不用正则,不到最后不切片也不用中间变量存储,只在序列中标记当前查找字段的开头结尾,到返回时才切片,高效快速准确切割规则 + * 解决jsonPath自带的"&&"和"||"与阅读的规则冲突,以及规则正则或字符串中包含"&&"、"||"、"%%"、"@"导致的冲突 */ - private fun getElementsSingle(temp: Element, rule: String): Elements { + tailrec fun splitRule(vararg split: String): Array{ //首段匹配,elementsType为空 - var elements = Elements() + if(split.size == 1) { + elementsType = split[0] //设置分割字串 + step = elementsType.length //设置分隔符长度 + return splitRule(arrayOf()) //仅一个分隔字串时，直接二段解析更快 + }else if (!consumeToAny(* split)) return arrayOf(queue) //未找到分隔符 - val fi = findIndexSet(rule) //执行索引列表处理器 + val st = findToAny( '[','(' ) //查找筛选器 - val (filterType,ruleStr) = fi //获取操作类型及非索引部分的规则字串 + if(st == -1) { -// val rulePc = rulePcx[0].trim { it <= ' ' }.split(">") -// jsoup中，当前节点是参与选择的，tag.div 与 tag.div@tag.div 结果相同 -// 此处">"效果和“@”完全相同，且容易让人误解成选择子节点，实际并不是。以后不允许这种无意义的写法 + var rule = arrayOf(queue.substring(0, pos)) //压入分隔的首段规则到数组 - val rules = ruleStr.split(".") + elementsType = queue.substring(pos, pos + step) //设置组合类型 + pos += step //跳过分隔符 - elements.addAll( - if(ruleStr.isEmpty()) temp.children() //允许索引直接作为根元素，此时前置规则为空，效果与children相同 - else when (rules[0]) { - "children" -> temp.children() //允许索引直接作为根元素，此时前置规则为空，效果与children相同 - "class" -> temp.getElementsByClass(rules[1]) - "tag" -> temp.getElementsByTag(rules[1]) - "id" -> Collector.collect(Evaluator.Id(rules[1]), temp) - "text" -> temp.getElementsContainingOwnText(rules[1]) - else -> temp.select(ruleStr) - } ) + while (consumeToAny(* split)) { //循环切分规则压入数组 + rule += queue.substring(start, pos) + pos += step //跳过分隔符 + } - val indexSet = fi.getIndexs(elements.size) //传入元素数量，处理负数索引及索引越界问题，生成可用索引集合。 + rule += queue.substring(pos) //将剩余字段压入数组末尾 - if(filterType == '!'){ //排除 + return rule + } - for (pcInt in indexSet) elements[pcInt] = null + val rule = if(st >pos ){ //先匹配到st1pos，表明分隔字串不在选择器中，将选择器前分隔字串分隔的字段依次压入数组 - elements.removeAll(listOf(null)) //测试过，这样就行 + var rule = arrayOf(queue.substring(0, pos)) //压入分隔的首段规则到数组 - }else if(filterType == '.'){ //选择 + elementsType = queue.substring(pos, pos + step) //设置组合类型 + pos += step //跳过分隔符 - val es = Elements() + while (consumeToAny( * split ) && pos < st ) { //循环切分规则压入数组 + rule += queue.substring(start, pos) + pos += step //跳过分隔符 + } - for (pcInt in indexSet) es.add(elements[pcInt]) + rule - elements = es + }else null - } + pos = st //位置推移到筛选器处 + val next = if(queue[pos] == '[' ) ']' else ')' //平衡组末尾字符 - return elements - } + return if (rule == null) { //rule为空,首段未匹配完成 - /** - * 获取内容列表 - */ - private fun getResultList(ruleStr: String): List? { + if(!chompRuleBalanced(queue[pos],next)) throw Error(queue.substring(0, start)+"后未平衡") //拉出一个筛选器,不平衡则报错 + splitRule(* split) //递归调用首段匹配 - if (ruleStr.isEmpty()) return null + } else { - var elements = Elements() + val start0 = start //记录当前规则开头位置 + if(!chompRuleBalanced(queue[pos],next)) throw Error(queue.substring(0, start)+"后未平衡") //拉出一个筛选器,不平衡则报错 + start = start0 //筛选器的开头不是本段规则开头,故恢复开头设置 + splitRule(rule) //首段已匹配,但当前段匹配未完成,调用二段匹配 - elements.add(element) + } + + } - val rule = RuleAnalyzer(ruleStr) //创建解析 + @JvmName("splitRuleNext") + private tailrec fun splitRule(rules:Array): Array{ //二段匹配被调用,elementsType非空(已在首段赋值),直接按elementsType查找,比首段采用的方式更快 - while( rule.peek() =='@' || rule.peek() < '!' ) rule.advance() // 修剪当前规则之前的"@"或者空白符 + if (!consumeTo(elementsType,false)) return rules + queue.substring(start) //此处consumeTo(...)开始位置不是规则的开始位置,start沿用上次设置 - val rules = rule.splitRule("@") // 切割成列表 + val st = findToAny( '[','(' ) //查找筛选器 - val last = rules.size - 1 - for (i in 0 until last) { - val es = Elements() - for (elt in elements) { - es.addAll(getElementsSingle(elt, rules[i])) + if(st == -1) { + var rule = rules + queue.substring(start, pos) //压入本次分隔的首段规则到数组 + pos += step //跳过分隔符 + while (consumeTo(elementsType)) { //循环切分规则压入数组 + rule += queue.substring(start, pos) + pos += step //跳过分隔符 } - elements.clear() - elements = es + rule += queue.substring(pos) //将剩余字段压入数组末尾 + return rule } - return if (elements.isEmpty()) null else getResultLast(elements, rules[last]) - } - /** - * 根据最后一个规则获取内容 - */ - private fun getResultLast(elements: Elements, lastRule: String): List { - val textS = ArrayList() - try { - when (lastRule) { - "text" -> for (element in elements) { - textS.add(element.text()) - } - "textNodes" -> for (element in elements) { - val tn = arrayListOf() - val contentEs = element.textNodes() - for (item in contentEs) { - val temp = item.text().trim { it <= ' ' } - if (temp.isNotEmpty()) { - tn.add(temp) - } - } - textS.add(join("\n", tn)) - } - "ownText" -> for (element in elements) { - textS.add(element.ownText()) - } - "html" -> { - elements.select("script").remove() - elements.select("style").remove() - val html = elements.outerHtml() - textS.add(html) - } - "all" -> textS.add(elements.outerHtml()) - else -> for (element in elements) { + val rule = if(st > pos ){//先匹配到st1pos，表明分隔字串不在选择器中，将选择器前分隔字串分隔的字段依次压入数组 + var rule = rules + queue.substring(start, pos) //压入本次分隔的首段规则到数组 + pos += step //跳过分隔符 + while (consumeTo(elementsType) && pos < st) { //循环切分规则压入数组 + rule += queue.substring(start, pos) + pos += step //跳过分隔符 + } + rule + }else rules - val url = element.attr(lastRule) + pos = st //位置推移到筛选器处 + val next = if(queue[pos] == '[' ) ']' else ')' //平衡组末尾字符 - if(url.isEmpty() || textS.contains(url)) break + val start0 = start //记录当前规则开头位置 + if(!chompRuleBalanced(queue[pos],next)) throw Error(queue.substring(0, start)+"后未平衡") //拉出一个筛选器,不平衡时返回true,表示未平衡 + start = start0 //筛选器平衡,但筛选器的开头不是当前规则开头,故恢复开头设置 - textS.add(url) - } - } - } catch (e: Exception) { - e.printStackTrace() - } + return splitRule(rule) //递归匹配 - return textS } - data class IndexSet(var split:Char = '.', - var beforeRule:String = "", - val indexDefault:MutableList = mutableListOf(), - val indexs:MutableList = mutableListOf()){ - fun getIndexs(len:Int): MutableSet { + /** + * 替换内嵌规则 + * @param inner 起始标志,如{$. 或 {{ + * @param startStep 不属于规则部分的前置字符长度，如{$.中{不属于规则的组成部分，故startStep为1 + * @param endStep 不属于规则部分的后置字符长度，如}}长度为2 + * @param fr 查找到内嵌规则时，用于解析的函数 + * + * */ + fun innerRule( inner:String,startStep:Int = 1,endStep:Int = 1,fr:(String)->String?): String { - val indexSet = mutableSetOf() + val start0 = pos //规则匹配前起点 - val lastIndexs = (indexDefault.size - 1).takeIf { it !=-1 } ?: indexs.size -1 + val st = StringBuilder() + while (!isEmpty && consumeTo(inner)) { //拉取成功返回true，ruleAnalyzes里的字符序列索引变量pos后移相应位置，否则返回false,且isEmpty为true - if(indexs.isEmpty())for (ix in lastIndexs downTo 0 ){ //indexs为空，表明是非[]式索引，集合是逆向遍历插入的，所以这里也逆向遍历，好还原顺序 + val start1 = start //记录拉取前起点 - val it = indexDefault[ix] - if(it in 0 until len) indexSet.add(it) //将正数不越界的索引添加到集合 - else if(it < 0 && len >= -it) indexSet.add(it + len) //将负数不越界的索引添加到集合 + if (chompRuleBalanced {//拉出一个以[]为默认嵌套、以{}为补充嵌套的平衡字段 + when (it) { + '{' -> true + '}' -> false + else -> null + } + }) { + val frv= fr(currBalancedString(startStep,endStep)) + if(frv != null) { + + st.append(queue.substring(start1,start)+frv) //压入内嵌规则前的内容，及内嵌规则解析得到的字符串 + continue //获取内容成功，继续选择下个内嵌规则 - }else for (ix in lastIndexs downTo 0 ){ //indexs不空，表明是[]式索引，集合是逆向遍历插入的，所以这里也逆向遍历，好还原顺序 + } + } - if(indexs[ix] is Triple<*, *, *>){ //区间 + start = start1 //拉出字段不平衡，重置起点 + pos = start + inner.length //拉出字段不平衡，inner只是个普通字串，规则回退到开头，并跳到此inner后继续匹配 - val (startx, endx, stepx) = indexs[ix] as Triple //还原储存时的类型 + } - val start = if (startx == null) 0 //左端省略表示0 - else if (startx >= 0) if (startx < len) startx else len - 1 //右端越界，设置为最大索引 - else if (-startx <= len) len + startx /* 将负索引转正 */ else 0 //左端越界，设置为最小索引 + //匹配前起点与当前规则起点相同，证明无替换成功的内嵌规则,返回空字符串。否则返回替换后的字符串 + return if(start0 == start) "" else { + st.append(remainingString()) //压入剩余字符串 + st.toString() + } + } - val end = if (endx == null) len - 1 //右端省略表示 len - 1 - else if (endx >= 0) if (endx < len) endx else len - 1 //右端越界，设置为最大索引 - else if (-endx <= len) len + endx /* 将负索引转正 */ else 0 //左端越界，设置为最小索引 +// /** +// * 匹配并返回标签中的属性键字串（字母、数字、-、_、:） +// * @return 属性键字串 +// */ +// fun consumeAttributeKey(start:Int = pos): String { +// while (!isEmpty && (Character.isLetterOrDigit(queue[pos]) || matchesAny('-', '_', ':'))) pos++ +// return queue.substring(start, pos) +// } + +// fun splitRule(query:String,item:String = "other",listItem:String = "allInOne"):String{ +// +// val cuurItem = item //当前项类型，list->列表项 mulu->章节列表项 url->链接项 search->搜索链接项 find发现链接列表项 other->其他项 +// val cuurList = listItem//当前界面总列表项类型，allInOne，json，xml，kotin，java +// var Reverse = false //是否反转列表 +// +// consumeWhitespace() //消耗开头空白 +// var fisrt = consume() //拉出并消费首字符 +// +// when(item){ +// "search" -> +// "find" -> +// "mulu" -> if(fisrt == '-'){ +// Reverse=true //开启反转 +// consumeWhitespace() //拉出所有空白符 +// fisrt = consume() //首字符后移 +// } +// else -> +// +// } +// +// return query +// } - if (start == end || stepx >= len) { //两端相同，区间里只有一个数。或间隔过大，区间实际上仅有首位 + companion object { + /** + * 转义字符 + */ + private const val ESC = '\\' - indexSet.add(start) - continue + /** + * 阅读共有分隔字串起始部分 + * "##","@@","{{","{[","", "@js:" + */ + val splitList =arrayOf("##","@@","{{","{[","", "@js:") - } + /** + * 发现‘名称-链接’分隔字串 + * "::" + */ + const val splitListFaXian = "::" - val step = if (stepx > 0) stepx else if (-stepx < len) stepx + len else 1 //最小正数间隔为1 + /** + * 目录专有起始字符 + * "-" + */ + const val splitListMulu = "-" - //将区间展开到集合中,允许列表反向。 - indexSet.addAll(if (end > start) start..end step step else start downTo end step step) + /** + * 结果为元素列表的 all in one 模式起始字符 + * "+" + */ + const val splitListTongYi = "+" - }else{//单个索引 + /** + * 结果为元素列表的项的同规则组合结构 + * "||","&&","%%" + */ + val splitListReSplit = arrayOf("||","&&","%%") - val it = indexs[ix] as Int //还原储存时的类型 + /** + * js脚本结束字串 + * "" + */ + const val splitListEndJS = "" - if(it in 0 until len) indexSet.add(it) //将正数不越界的索引添加到集合 - else if(it < 0 && len >= -it) indexSet.add(it + len) //将负数不越界的索引添加到集合 + /** + *内嵌js结束字串 + * "}}" + */ + const val splitListEndInnerJS = "}}" - } + /** + * 内嵌规则结束字串 + * "]}" + */ + const val splitListEndInnerRule = "]}" - } + /** + * '[', ']', '(', ')','{','}' + */ + val splitListPublic = charArrayOf('[', ']', '(', ')','{','}') - return indexSet + /** + * '*',"/","//",":","::","@","|","@xpath:" + */ + val splitListXpath = arrayOf("*","/","//",":","::","@","|","@xpath:") - } + /** + * '*','$',".","..", "@json:" + */ + val splitListJson = arrayOf('*','$',".","..", "@json:") - } + /** + * '*',"+","~",".",",","|","@","@css:",":" + */ + val splitListCss = arrayOf('*',"+","~",".",",","|","@","@css:",":") + /** + * "-",".","!","@","@@" + */ + val splitListDefault = arrayOf("-",".","!","@","@@") - internal inner class SourceRule(ruleStr: String) { - var isCss = false - var elementsRule: String = if (ruleStr.startsWith("@CSS:", true)) { - isCss = true - ruleStr.substring(5).trim { it <= ' ' } - } else { - ruleStr - } } - } From e8bc90fd9f46af43d6f6ab7cbe31d86e8ca8cdc1 Mon Sep 17 00:00:00 2001 From: bushixuanqi <57338301+bushixuanqi@users.noreply.github.com> Date: Sun, 4 Jul 2021 22:43:19 +0800 Subject: [PATCH 2/3] Revert AnalyzeByJSoup.kt MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 复制错了，回退 --- .../app/model/analyzeRule/AnalyzeByJSoup.kt | 773 +++++++++--------- 1 file changed, 372 insertions(+), 401 deletions(-) diff --git a/app/src/main/java/io/legado/app/model/analyzeRule/AnalyzeByJSoup.kt b/app/src/main/java/io/legado/app/model/analyzeRule/AnalyzeByJSoup.kt index dfc5cae23..3bb2e4e33 100644 --- a/app/src/main/java/io/legado/app/model/analyzeRule/AnalyzeByJSoup.kt +++ b/app/src/main/java/io/legado/app/model/analyzeRule/AnalyzeByJSoup.kt @@ -1,525 +1,496 @@ package io.legado.app.model.analyzeRule -//通用的规则切分处理 - -class RuleAnalyzer(data: String) { - - private var queue: String = data //被处理字符串 - private var pos = 0 //处理到的位置 - - private var start = 0 //每次处理字段的开始 - private var step:Int = 0 //分割字符的长度 - - var elementsType = "" - - //当前平衡字段 - fun currBalancedString( stepStart:Int = 1 , stepEnd:Int = 1): String { //stepStart平衡字符的起始分隔字串长度，stepEnd平衡字符的结束分隔字串长度 - return queue.substring(start+stepStart,pos-stepEnd) //当前平衡字段 - } +import android.text.TextUtils.join +import androidx.annotation.Keep +import org.jsoup.Jsoup +import org.jsoup.nodes.Element +import org.jsoup.select.Collector +import org.jsoup.select.Elements +import org.jsoup.select.Evaluator +import org.seimicrawler.xpath.JXNode +import java.util.* + +/** + * Created by GKF on 2018/1/25. + * 书源规则解析 + */ +@Keep +class AnalyzeByJSoup(doc: Any) { + companion object { + /** + * "class", "id", "tag", "text", "children" + */ + val validKeys = arrayOf("class", "id", "tag", "text", "children") - //将pos重置为0，方便复用 - fun reSetPos() { - pos = 0 - } + fun parse(doc: Any): Element { + return when (doc) { + is Element -> doc + is JXNode -> if (doc.isElement) doc.asElement() else Jsoup.parse(doc.toString()) + else -> Jsoup.parse(doc.toString()) + } + } - //当前拉取字段 - fun currString(): String { - return queue.substring(start,pos) //当前拉取到的字段 } - //剩余字串 - private fun remainingString(): String { - start = pos - pos = queue.length - return queue.substring(start) - } + private var element: Element = parse(doc) /** - * pos位置回退 + * 获取列表 */ - fun back(num :Int = 0) { - if(num == 0)pos = start //回退 - else pos -= num - } + internal fun getElements(rule: String) = getElements(element, rule) /** - * pos位置后移 + * 合并内容列表,得到内容 */ - fun advance(num :Int = 1) { - pos+=num - } + internal fun getString(ruleStr: String) = + if(ruleStr.isEmpty()) null + else getStringList(ruleStr).takeIf { it.isNotEmpty() }?.joinToString("\n") /** - * 是否已无剩余字符? - * @return 若剩余字串中已无字符则返回true + * 获取一个字符串 */ - val isEmpty: Boolean - get() = queue.length - pos == 0 //是否处理到最后 + internal fun getString0(ruleStr: String) = getStringList(ruleStr).let{ if ( it.isEmpty() ) "" else it[0] } /** - * 检索并返回首字符,但pos不变 - * @return 首字符：若为空则为 0 号字符 + * 获取所有内容列表 */ - fun peek(): Char { //检索首字符 - return if (isEmpty) 0.toChar() else queue[pos] - } + internal fun getStringList(ruleStr: String): List { - /** - * 消耗剩余字串中一个字符。 - * @return 返回剩余字串中的下个字符。 - */ - fun consume(): Char { - return queue[pos++] - } + val textS = ArrayList() - /** - * 字串与剩余字串是否匹配，不区分大小写 - * @param seq 字符串被检查 - * @return 若下字符串匹配返回 true - */ - fun matches(seq: String): Boolean { - return queue.regionMatches(pos, seq, 0, seq.length, ignoreCase = true) - } + if (ruleStr.isEmpty()) return textS - /** - * 从剩余字串中拉出一个字符串，直到但不包括匹配序列，或剩余字串用完。 - * @param seq 分隔字符 **区分大小写** - * @return 是否找到相应字段。 - */ - fun consumeTo(seq: String,setStartPos:Boolean = true): Boolean { + //拆分规则 + val sourceRule = SourceRule(ruleStr) + + if (sourceRule.elementsRule.isEmpty()) { + + textS.add(element.data() ?: "") + + } else { + + val ruleAnalyzes = RuleAnalyzer(sourceRule.elementsRule) + val ruleStrS = ruleAnalyzes.splitRule("&&","||" ,"%%") + + val results = ArrayList>() + for (ruleStrX in ruleStrS) { + + val temp: List? = + if (sourceRule.isCss) { + val lastIndex = ruleStrX.lastIndexOf('@') + getResultLast( + element.select(ruleStrX.substring(0, lastIndex)), + ruleStrX.substring(lastIndex + 1) + ) + } else { + getResultList(ruleStrX) + } + + if (!temp.isNullOrEmpty()) { + + results.add(temp) //!temp.isNullOrEmpty()时，results.isNotEmpty()为true - if(setStartPos)start = pos //将处理到的位置设置为规则起点 - val offset = queue.indexOf(seq, pos) - return if (offset != -1) { - pos = offset - true - } else false + if (ruleAnalyzes.elementsType == "||") break + } + } + if (results.size > 0) { + if ("%%" == ruleAnalyzes.elementsType) { + for (i in results[0].indices) { + for (temp in results) { + if (i < temp.size) { + textS.add(temp[i]) + } + } + } + } else { + for (temp in results) { + textS.addAll(temp) + } + } + } + } + return textS } /** - * 从剩余字串中拉出一个字符串，直到但不包括匹配序列（匹配参数列表中一项即为匹配），或剩余字串用完。 - * @param seq 匹配字符串序列 - * @return 成功返回true并设置间隔，失败则直接返回fasle + * 获取Elements */ - fun consumeToAny(vararg seq:String): Boolean { + private fun getElements(temp: Element?, rule: String): Elements { - start = pos + if (temp == null || rule.isEmpty()) return Elements() - while (!isEmpty) { + val elements = Elements() - for (s in seq) { - if (matches(s)) { - step = s.length //间隔数 - return true //匹配就返回 true + val sourceRule = SourceRule(rule) + val ruleAnalyzes = RuleAnalyzer(sourceRule.elementsRule) + val ruleStrS = ruleAnalyzes.splitRule("&&","||","%%") + + val elementsList = ArrayList() + if (sourceRule.isCss) { + for (ruleStr in ruleStrS) { + val tempS = temp.select(ruleStr) + elementsList.add(tempS) + if (tempS.size > 0 && ruleAnalyzes.elementsType == "||") { + break } } + } else { + for (ruleStr in ruleStrS) { + //将原getElementsSingle函数调用的函数的部分代码内联过来，方便简化getElementsSingle函数 + + val rsRule = RuleAnalyzer(ruleStr) + + if( rsRule.peek() =='@' || rsRule.peek() < '!' ) rsRule.advance() // 修剪当前规则之前的"@"或者空白符 + + val rs = rsRule.splitRule("@") + + val el = if (rs.size > 1) { + val el = Elements() + el.add(temp) + for (rl in rs) { + val es = Elements() + for (et in el) { + es.addAll(getElements(et, rl)) + } + el.clear() + el.addAll(es) + } + el + }else getElementsSingle(temp,ruleStr) - pos++ //逐个试探 + elementsList.add(el) + if (el.size > 0 && ruleAnalyzes.elementsType == "||") { + break + } + } } - - pos = start //匹配失败，位置回退 - - return false + if (elementsList.size > 0) { + if ("%%" == ruleAnalyzes.elementsType) { + for (i in 0 until elementsList[0].size) { + for (es in elementsList) { + if (i < es.size) { + elements.add(es[i]) + } + } + } + } else { + for (es in elementsList) { + elements.addAll(es) + } + } + } + return elements } /** - * 从剩余字串中拉出一个字符串，直到但不包括匹配序列（匹配参数列表中一项即为匹配），或剩余字串用完。 - * @param seq 匹配字符序列 - * @return 返回匹配位置 - */ - private fun findToAny(vararg seq:Char): Int { + * 1.支持阅读原有写法，':'分隔索引，!或.表示筛选方式，索引可为负数 + * + * 例如 tag.div.-1:10:2 或 tag.div!0:3 + * + * 2. 支持与jsonPath类似的[]索引写法 + * + * 格式形如 [it,it，。。。] 或 [!it,it，。。。] 其中[!开头表示筛选方式为排除，it为单个索引或区间。 + * + * 区间格式为 start:end 或 start:end:step，其中start为0可省略，end为-1可省略。 + * + * 索引，区间两端及间隔都支持负数 + * + * 例如 tag.div[-1, 3:-2:-10, 2] + * + * 特殊用法 tag.div[-1:0] 可在任意地方让列表反向 + * + * */ - val start = pos //声明新变量记录临时起始位置，不更改类本身的起始位置 + fun findIndexSet( rule:String ): IndexSet { - while (!isEmpty) { + val indexSet = IndexSet() + val rus = rule.trim{ it <= ' '} - for (s in seq) if(queue[pos] == s) return pos //匹配则返回位置 + var len = rus.length + var curInt: Int? //当前数字 + var curMinus = false //当前数字是否为负 + val curList = mutableListOf() //当前数字区间 + var l = "" //暂存数字字符串 - pos++ //逐个试探 + val head = rus.last() == ']' //是否为常规索引写法 - } + if(head){ //常规索引写法[index...] - pos = start //匹配失败，位置回退 + len-- //跳过尾部']' - return -1 - } + while (len-- >= 0) { //逆向遍历,可以无前置规则 - //其中js只要符合语法，就不用避开任何阅读关键字，自由发挥 - fun chompJsBalanced(f: ((Char) -> Boolean?) = { - when (it) { - '{' -> true //开始嵌套一层 - '}' -> false //闭合一层嵌套 - else -> null - } - } ): Boolean { - start = pos - var depth = 0 //嵌套深度 - var bracketsDepth = 0 //[]嵌套深度 - - var inSingleQuote = false //单引号 - var inDoubleQuote = false //双引号 - var inOtherQuote = false //js原始字串分隔字符 - var regex = false //正则 - var commit = false //单行注释 - var commits = false //多行注释 - - do { - if (isEmpty) break - var c = consume() - if (c != '\\') { //非转义字符 - if (c == '\'' && !commits && !commit && !regex && !inDoubleQuote && !inOtherQuote) inSingleQuote = !inSingleQuote //匹配具有语法功能的单引号 - else if (c == '"' && !commits && !commit && !regex && !inSingleQuote && !inOtherQuote) inDoubleQuote = !inDoubleQuote //匹配具有语法功能的双引号 - else if (c == '`' && !commits && !commit && !regex && !inSingleQuote && !inDoubleQuote) inOtherQuote = !inOtherQuote //匹配具有语法功能的'`' - else if (c == '/' && !commits && !commit && !regex && !inSingleQuote && !inDoubleQuote && !inOtherQuote) { //匹配注释或正则起点 - c = consume() - when(c){ - '/'->commit=true //匹配单行注释起点 - '*'->commits=true //匹配多行注释起点 - else ->regex=true //匹配正则起点 - } - } - else if(commits && c == '*') { //匹配多行注释终点 - c = consume() - if(c == '/')commits = false - } - else if(regex && c == '/') { //正则的终点或[]平衡 + var rl = rus[len] + if (rl == ' ') continue //跳过空格 - when (c) { - '/' -> regex = false//匹配正则终点 + if (rl in '0'..'9') l += rl //将数值累接入临时字串中，遇到分界符才取出 + else if (rl == '-') curMinus = true + else { - //为了保证当open为（ 且 close 为 ）时，正则中[(]或[)]的合法性。故对[]这对在任何规则中都平衡的成对符号做匹配。 - // 注：正则里[(]、[)]、[{]、[}]都是合法的，所以只有[]必须平衡。 - '[' -> bracketsDepth++ //开始嵌套一层[] - ']' -> bracketsDepth-- //闭合一层嵌套[] - } + curInt = if (l.isEmpty()) null else if (curMinus) -l.toInt() else l.toInt() //当前数字 - } + when (rl) { + + ':' -> curList.add(curInt) //区间右端或区间间隔 - if (commits || commit || regex || inSingleQuote || inDoubleQuote || inOtherQuote) continue //语法单元未匹配结束，直接进入下个循环 + else -> { - val fn = f(c) ?: continue - if (fn) depth++ else depth-- //嵌套或者闭合 + //为保证查找顺序，区间和单个索引都添加到同一集合 + if(curList.isEmpty())indexSet.indexs.add(curInt!!) + else{ - }else { //转义字符 - var next = consume() //拉出被转义字符 - if(commit && next == 'n') commit = false //匹配单行注释终点。当前为\,下个为n，表示换行 - else if (!commits && !commit && next == '\\') { - consume() //当前为\,下个为\，双重转义中"\\"表示转义字符本身，根据if条件"\\"字串不在注释中，则只能在字串或正则中 - next = consume() //拉出下个字符，因为在双重转义的字串或正则中，类似于 \\/ 这样的结构才是转义结构 - if(next == '\\')consume() //若为转义字符则继续拉出，因为双重转义中转义字符成对存在,即 \\\\ + //列表最后压入的是区间右端，若列表有两位则最先压入的是间隔 + indexSet.indexs.add( Triple(curInt, curList.last(), if(curList.size == 2) curList.first() else 1) ) + + curList.clear() //重置临时列表，避免影响到下个区间的处理 + + } + + if(rl == '!'){ + indexSet.split='!' + do{ rl = rus[--len] } while (len > 0 && rl == ' ')//跳过所有空格 + } + + if(rl == '[') return indexSet.apply { + beforeRule = rus.substring(0, len) + } //遇到索引边界，返回结果 + + if(rl != ',') break //非索引结构，跳出 + + } + } + + l = "" //清空 + curMinus = false //重置 } } - } while (depth > 0 || bracketsDepth >0) //拉出全部符合js语法的字段 + } else while (len-- >= 0) { //阅读原本写法，逆向遍历,可以无前置规则 - if(depth > 0 || bracketsDepth >0) start = pos + val rl = rus[len] + if (rl == ' ') continue //跳过空格 - return pos > start - } + if (rl in '0'..'9') l += rl //将数值累接入临时字串中，遇到分界符才取出 + else if (rl == '-') curMinus = true + else { - /** - * 在双重转义字串中拉出一个规则平衡组 - */ - fun chompRuleBalanced(open: Char = '[', close: Char = ']',f: ((Char) ->Boolean?)? = null ): Boolean { - start = pos - var depth = 0 //嵌套深度 - var otherDepth = 0 //其他对称符合嵌套深度 - - var inSingleQuote = false //单引号 - var inDoubleQuote = false //双引号 - - do { - if (isEmpty) break - val c = consume() - if (c != ESC) { //非转义字符 - if (c == '\'' && !inDoubleQuote) inSingleQuote = !inSingleQuote //匹配具有语法功能的单引号 - else if (c == '"' && !inSingleQuote) inDoubleQuote = !inDoubleQuote //匹配具有语法功能的双引号 - - if (inSingleQuote || inDoubleQuote) continue //语法单元未匹配结束，直接进入下个循环 - - if ( c == open )depth++ //开始嵌套一层 - else if ( c== close) depth-- //闭合一层嵌套 - else if(depth == 0 && f != null) { //处于默认嵌套中的非默认字符不需要平衡，仅depth为0时默认嵌套全部闭合，此字符才进行嵌套 - val fn = f(c) ?: continue - if (fn) otherDepth++ else otherDepth-- - } + if(rl == '!' || rl == '.' || rl == ':') { //分隔符或起始符 - }else { //转义字符 - var next = consume() //拉出被转义字符，匹配\/、\"、\'等 - if (next == ESC) { - consume() //当前为\,下个为\，双重转义中"\\"表示转义字符本身，根据语法特征当前字段在字串或正则中 - next = consume() //拉出下个字符，因为在双重转义的字串或正则中，类似于 \\/ 这样的结构才是转义结构 - if(next == ESC)consume() //若为转义字符则继续拉出，因为双重转义中转义字符成对存在,即 \\\\ - } + indexSet.indexDefault.add(if (curMinus) -l.toInt() else l.toInt()) // 当前数字追加到列表 + + if (rl != ':') return indexSet.apply { //rl == '!' || rl == '.' + split = rl + beforeRule = rus.substring(0, len) + } + + }else break //非索引结构，跳出循环 + + l = "" //清空 + curMinus = false //重置 } - } while (depth > 0 || otherDepth > 0) //拉出一个平衡字串 - return !(depth > 0 || otherDepth > 0) //平衡返回false，不平衡返回true + } + + return indexSet.apply{ + split = ' ' + beforeRule = rus } //非索引格式 } /** - * 不用正则,不到最后不切片也不用中间变量存储,只在序列中标记当前查找字段的开头结尾,到返回时才切片,高效快速准确切割规则 - * 解决jsonPath自带的"&&"和"||"与阅读的规则冲突,以及规则正则或字符串中包含"&&"、"||"、"%%"、"@"导致的冲突 + * 获取Elements按照一个规则 */ - tailrec fun splitRule(vararg split: String): Array{ //首段匹配,elementsType为空 + private fun getElementsSingle(temp: Element, rule: String): Elements { - if(split.size == 1) { - elementsType = split[0] //设置分割字串 - step = elementsType.length //设置分隔符长度 - return splitRule(arrayOf()) //仅一个分隔字串时，直接二段解析更快 - }else if (!consumeToAny(* split)) return arrayOf(queue) //未找到分隔符 + var elements = Elements() - val st = findToAny( '[','(' ) //查找筛选器 + val fi = findIndexSet(rule) //执行索引列表处理器 - if(st == -1) { + val (filterType,ruleStr) = fi //获取操作类型及非索引部分的规则字串 - var rule = arrayOf(queue.substring(0, pos)) //压入分隔的首段规则到数组 +// val rulePc = rulePcx[0].trim { it <= ' ' }.split(">") +// jsoup中，当前节点是参与选择的，tag.div 与 tag.div@tag.div 结果相同 +// 此处">"效果和“@”完全相同，且容易让人误解成选择子节点，实际并不是。以后不允许这种无意义的写法 - elementsType = queue.substring(pos, pos + step) //设置组合类型 - pos += step //跳过分隔符 + val rules = ruleStr.split(".") - while (consumeToAny(* split)) { //循环切分规则压入数组 - rule += queue.substring(start, pos) - pos += step //跳过分隔符 - } + elements.addAll( + if(ruleStr.isEmpty()) temp.children() //允许索引直接作为根元素，此时前置规则为空，效果与children相同 + else when (rules[0]) { + "children" -> temp.children() //允许索引直接作为根元素，此时前置规则为空，效果与children相同 + "class" -> temp.getElementsByClass(rules[1]) + "tag" -> temp.getElementsByTag(rules[1]) + "id" -> Collector.collect(Evaluator.Id(rules[1]), temp) + "text" -> temp.getElementsContainingOwnText(rules[1]) + else -> temp.select(ruleStr) + } ) - rule += queue.substring(pos) //将剩余字段压入数组末尾 + val indexSet = fi.getIndexs(elements.size) //传入元素数量，处理负数索引及索引越界问题，生成可用索引集合。 - return rule - } + if(filterType == '!'){ //排除 - val rule = if(st >pos ){ //先匹配到st1pos，表明分隔字串不在选择器中，将选择器前分隔字串分隔的字段依次压入数组 + for (pcInt in indexSet) elements[pcInt] = null - var rule = arrayOf(queue.substring(0, pos)) //压入分隔的首段规则到数组 + elements.removeAll(listOf(null)) //测试过，这样就行 - elementsType = queue.substring(pos, pos + step) //设置组合类型 - pos += step //跳过分隔符 + }else if(filterType == '.'){ //选择 - while (consumeToAny( * split ) && pos < st ) { //循环切分规则压入数组 - rule += queue.substring(start, pos) - pos += step //跳过分隔符 - } + val es = Elements() - rule + for (pcInt in indexSet) es.add(elements[pcInt]) - }else null + elements = es - pos = st //位置推移到筛选器处 - val next = if(queue[pos] == '[' ) ']' else ')' //平衡组末尾字符 - - return if (rule == null) { //rule为空,首段未匹配完成 + } - if(!chompRuleBalanced(queue[pos],next)) throw Error(queue.substring(0, start)+"后未平衡") //拉出一个筛选器,不平衡则报错 - splitRule(* split) //递归调用首段匹配 + return elements + } - } else { + /** + * 获取内容列表 + */ + private fun getResultList(ruleStr: String): List? { - val start0 = start //记录当前规则开头位置 - if(!chompRuleBalanced(queue[pos],next)) throw Error(queue.substring(0, start)+"后未平衡") //拉出一个筛选器,不平衡则报错 - start = start0 //筛选器的开头不是本段规则开头,故恢复开头设置 - splitRule(rule) //首段已匹配,但当前段匹配未完成,调用二段匹配 + if (ruleStr.isEmpty()) return null - } + var elements = Elements() - } + elements.add(element) - @JvmName("splitRuleNext") - private tailrec fun splitRule(rules:Array): Array{ //二段匹配被调用,elementsType非空(已在首段赋值),直接按elementsType查找,比首段采用的方式更快 + val rule = RuleAnalyzer(ruleStr) //创建解析 - if (!consumeTo(elementsType,false)) return rules + queue.substring(start) //此处consumeTo(...)开始位置不是规则的开始位置,start沿用上次设置 + while( rule.peek() =='@' || rule.peek() < '!' ) rule.advance() // 修剪当前规则之前的"@"或者空白符 - val st = findToAny( '[','(' ) //查找筛选器 + val rules = rule.splitRule("@") // 切割成列表 - if(st == -1) { - var rule = rules + queue.substring(start, pos) //压入本次分隔的首段规则到数组 - pos += step //跳过分隔符 - while (consumeTo(elementsType)) { //循环切分规则压入数组 - rule += queue.substring(start, pos) - pos += step //跳过分隔符 + val last = rules.size - 1 + for (i in 0 until last) { + val es = Elements() + for (elt in elements) { + es.addAll(getElementsSingle(elt, rules[i])) } - rule += queue.substring(pos) //将剩余字段压入数组末尾 - return rule + elements.clear() + elements = es } + return if (elements.isEmpty()) null else getResultLast(elements, rules[last]) + } - val rule = if(st > pos ){//先匹配到st1pos，表明分隔字串不在选择器中，将选择器前分隔字串分隔的字段依次压入数组 - var rule = rules + queue.substring(start, pos) //压入本次分隔的首段规则到数组 - pos += step //跳过分隔符 - while (consumeTo(elementsType) && pos < st) { //循环切分规则压入数组 - rule += queue.substring(start, pos) - pos += step //跳过分隔符 - } - rule - }else rules + /** + * 根据最后一个规则获取内容 + */ + private fun getResultLast(elements: Elements, lastRule: String): List { + val textS = ArrayList() + try { + when (lastRule) { + "text" -> for (element in elements) { + textS.add(element.text()) + } + "textNodes" -> for (element in elements) { + val tn = arrayListOf() + val contentEs = element.textNodes() + for (item in contentEs) { + val temp = item.text().trim { it <= ' ' } + if (temp.isNotEmpty()) { + tn.add(temp) + } + } + textS.add(join("\n", tn)) + } + "ownText" -> for (element in elements) { + textS.add(element.ownText()) + } + "html" -> { + elements.select("script").remove() + elements.select("style").remove() + val html = elements.outerHtml() + textS.add(html) + } + "all" -> textS.add(elements.outerHtml()) + else -> for (element in elements) { - pos = st //位置推移到筛选器处 - val next = if(queue[pos] == '[' ) ']' else ')' //平衡组末尾字符 + val url = element.attr(lastRule) - val start0 = start //记录当前规则开头位置 - if(!chompRuleBalanced(queue[pos],next)) throw Error(queue.substring(0, start)+"后未平衡") //拉出一个筛选器,不平衡时返回true,表示未平衡 - start = start0 //筛选器平衡,但筛选器的开头不是当前规则开头,故恢复开头设置 + if(url.isEmpty() || textS.contains(url)) break - return splitRule(rule) //递归匹配 + textS.add(url) + } + } + } catch (e: Exception) { + e.printStackTrace() + } + return textS } + data class IndexSet(var split:Char = '.', + var beforeRule:String = "", + val indexDefault:MutableList = mutableListOf(), + val indexs:MutableList = mutableListOf()){ - /** - * 替换内嵌规则 - * @param inner 起始标志,如{$. 或 {{ - * @param startStep 不属于规则部分的前置字符长度，如{$.中{不属于规则的组成部分，故startStep为1 - * @param endStep 不属于规则部分的后置字符长度，如}}长度为2 - * @param fr 查找到内嵌规则时，用于解析的函数 - * - * */ - fun innerRule( inner:String,startStep:Int = 1,endStep:Int = 1,fr:(String)->String?): String { + fun getIndexs(len:Int): MutableSet { - val start0 = pos //规则匹配前起点 + val indexSet = mutableSetOf() - val st = StringBuilder() + val lastIndexs = (indexDefault.size - 1).takeIf { it !=-1 } ?: indexs.size -1 - while (!isEmpty && consumeTo(inner)) { //拉取成功返回true，ruleAnalyzes里的字符序列索引变量pos后移相应位置，否则返回false,且isEmpty为true - val start1 = start //记录拉取前起点 + if(indexs.isEmpty())for (ix in lastIndexs downTo 0 ){ //indexs为空，表明是非[]式索引，集合是逆向遍历插入的，所以这里也逆向遍历，好还原顺序 - if (chompRuleBalanced {//拉出一个以[]为默认嵌套、以{}为补充嵌套的平衡字段 - when (it) { - '{' -> true - '}' -> false - else -> null - } - }) { - val frv= fr(currBalancedString(startStep,endStep)) - if(frv != null) { - - st.append(queue.substring(start1,start)+frv) //压入内嵌规则前的内容，及内嵌规则解析得到的字符串 - continue //获取内容成功，继续选择下个内嵌规则 + val it = indexDefault[ix] + if(it in 0 until len) indexSet.add(it) //将正数不越界的索引添加到集合 + else if(it < 0 && len >= -it) indexSet.add(it + len) //将负数不越界的索引添加到集合 - } - } + }else for (ix in lastIndexs downTo 0 ){ //indexs不空，表明是[]式索引，集合是逆向遍历插入的，所以这里也逆向遍历，好还原顺序 - start = start1 //拉出字段不平衡，重置起点 - pos = start + inner.length //拉出字段不平衡，inner只是个普通字串，规则回退到开头，并跳到此inner后继续匹配 + if(indexs[ix] is Triple<*, *, *>){ //区间 - } + val (startx, endx, stepx) = indexs[ix] as Triple //还原储存时的类型 - //匹配前起点与当前规则起点相同，证明无替换成功的内嵌规则,返回空字符串。否则返回替换后的字符串 - return if(start0 == start) "" else { - st.append(remainingString()) //压入剩余字符串 - st.toString() - } - } + val start = if (startx == null) 0 //左端省略表示0 + else if (startx >= 0) if (startx < len) startx else len - 1 //右端越界，设置为最大索引 + else if (-startx <= len) len + startx /* 将负索引转正 */ else 0 //左端越界，设置为最小索引 -// /** -// * 匹配并返回标签中的属性键字串（字母、数字、-、_、:） -// * @return 属性键字串 -// */ -// fun consumeAttributeKey(start:Int = pos): String { -// while (!isEmpty && (Character.isLetterOrDigit(queue[pos]) || matchesAny('-', '_', ':'))) pos++ -// return queue.substring(start, pos) -// } - -// fun splitRule(query:String,item:String = "other",listItem:String = "allInOne"):String{ -// -// val cuurItem = item //当前项类型，list->列表项 mulu->章节列表项 url->链接项 search->搜索链接项 find发现链接列表项 other->其他项 -// val cuurList = listItem//当前界面总列表项类型，allInOne，json，xml，kotin，java -// var Reverse = false //是否反转列表 -// -// consumeWhitespace() //消耗开头空白 -// var fisrt = consume() //拉出并消费首字符 -// -// when(item){ -// "search" -> -// "find" -> -// "mulu" -> if(fisrt == '-'){ -// Reverse=true //开启反转 -// consumeWhitespace() //拉出所有空白符 -// fisrt = consume() //首字符后移 -// } -// else -> -// -// } -// -// return query -// } + val end = if (endx == null) len - 1 //右端省略表示 len - 1 + else if (endx >= 0) if (endx < len) endx else len - 1 //右端越界，设置为最大索引 + else if (-endx <= len) len + endx /* 将负索引转正 */ else 0 //左端越界，设置为最小索引 - companion object { - /** - * 转义字符 - */ - private const val ESC = '\\' + if (start == end || stepx >= len) { //两端相同，区间里只有一个数。或间隔过大，区间实际上仅有首位 - /** - * 阅读共有分隔字串起始部分 - * "##","@@","{{","{[","", "@js:" - */ - val splitList =arrayOf("##","@@","{{","{[","", "@js:") + indexSet.add(start) + continue - /** - * 发现‘名称-链接’分隔字串 - * "::" - */ - const val splitListFaXian = "::" + } - /** - * 目录专有起始字符 - * "-" - */ - const val splitListMulu = "-" + val step = if (stepx > 0) stepx else if (-stepx < len) stepx + len else 1 //最小正数间隔为1 - /** - * 结果为元素列表的 all in one 模式起始字符 - * "+" - */ - const val splitListTongYi = "+" + //将区间展开到集合中,允许列表反向。 + indexSet.addAll(if (end > start) start..end step step else start downTo end step step) - /** - * 结果为元素列表的项的同规则组合结构 - * "||","&&","%%" - */ - val splitListReSplit = arrayOf("||","&&","%%") + }else{//单个索引 - /** - * js脚本结束字串 - * "" - */ - const val splitListEndJS = "" + val it = indexs[ix] as Int //还原储存时的类型 - /** - *内嵌js结束字串 - * "}}" - */ - const val splitListEndInnerJS = "}}" + if(it in 0 until len) indexSet.add(it) //将正数不越界的索引添加到集合 + else if(it < 0 && len >= -it) indexSet.add(it + len) //将负数不越界的索引添加到集合 - /** - * 内嵌规则结束字串 - * "]}" - */ - const val splitListEndInnerRule = "]}" + } - /** - * '[', ']', '(', ')','{','}' - */ - val splitListPublic = charArrayOf('[', ']', '(', ')','{','}') + } - /** - * '*',"/","//",":","::","@","|","@xpath:" - */ - val splitListXpath = arrayOf("*","/","//",":","::","@","|","@xpath:") + return indexSet - /** - * '*','$',".","..", "@json:" - */ - val splitListJson = arrayOf('*','$',".","..", "@json:") + } - /** - * '*',"+","~",".",",","|","@","@css:",":" - */ - val splitListCss = arrayOf('*',"+","~",".",",","|","@","@css:",":") + } - /** - * "-",".","!","@","@@" - */ - val splitListDefault = arrayOf("-",".","!","@","@@") + internal inner class SourceRule(ruleStr: String) { + var isCss = false + var elementsRule: String = if (ruleStr.startsWith("@CSS:", true)) { + isCss = true + ruleStr.substring(5).trim { it <= ' ' } + } else { + ruleStr + } } + } From 162cf0c6cdb1c5bbe77883a604c2b1b394db9b9f Mon Sep 17 00:00:00 2001 From: bushixuanqi <57338301+bushixuanqi@users.noreply.github.com> Date: Sun, 4 Jul 2021 22:48:20 +0800 Subject: [PATCH 3/3] Update RuleAnalyzer.kt MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 改进筛选器起始字符位置查找规则，改为逐个比对，先匹配到“(”就拉取“()”平衡，先匹配到“[”就拉取“[]”平衡，避免()和[]嵌套时出现边缘情况。 跳转筛选器匹配优先顺序，从“(”优先改为“[”优先。 --- .../legado/app/model/analyzeRule/RuleAnalyzer.kt | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/app/src/main/java/io/legado/app/model/analyzeRule/RuleAnalyzer.kt b/app/src/main/java/io/legado/app/model/analyzeRule/RuleAnalyzer.kt index 50d91ba15..dfc5cae23 100644 --- a/app/src/main/java/io/legado/app/model/analyzeRule/RuleAnalyzer.kt +++ b/app/src/main/java/io/legado/app/model/analyzeRule/RuleAnalyzer.kt @@ -83,7 +83,7 @@ class RuleAnalyzer(data: String) { /** * 从剩余字串中拉出一个字符串，直到但不包括匹配序列，或剩余字串用完。 - * @param seq ：分隔字符 **区分大小写** + * @param seq 分隔字符 **区分大小写** * @return 是否找到相应字段。 */ fun consumeTo(seq: String,setStartPos:Boolean = true): Boolean { @@ -130,14 +130,18 @@ class RuleAnalyzer(data: String) { */ private fun findToAny(vararg seq:Char): Int { - for (s in seq){ + val start = pos //声明新变量记录临时起始位置，不更改类本身的起始位置 - val offset = queue.indexOf(s, pos) + while (!isEmpty) { + + for (s in seq) if(queue[pos] == s) return pos //匹配则返回位置 - if (offset != -1) return offset //只要匹配到，就返回相应位置 + pos++ //逐个试探 } + pos = start //匹配失败，位置回退 + return -1 } @@ -265,7 +269,7 @@ class RuleAnalyzer(data: String) { return splitRule(arrayOf()) //仅一个分隔字串时，直接二段解析更快 }else if (!consumeToAny(* split)) return arrayOf(queue) //未找到分隔符 - val st = findToAny( '(','[' ) //查找筛选器 + val st = findToAny( '[','(' ) //查找筛选器 if(st == -1) { @@ -324,7 +328,7 @@ class RuleAnalyzer(data: String) { if (!consumeTo(elementsType,false)) return rules + queue.substring(start) //此处consumeTo(...)开始位置不是规则的开始位置,start沿用上次设置 - val st = findToAny( '(','[' ) //查找筛选器 + val st = findToAny( '[','(' ) //查找筛选器 if(st == -1) { var rule = rules + queue.substring(start, pos) //压入本次分隔的首段规则到数组