|
|
@ -137,11 +137,10 @@ class AnalyzeByJSoup(doc: Any) { |
|
|
|
} |
|
|
|
} |
|
|
|
} else { |
|
|
|
} else { |
|
|
|
for (ruleStr in ruleStrS) { |
|
|
|
for (ruleStr in ruleStrS) { |
|
|
|
//将原getElementsSingle函数调用的函数的部分代码内联过来,方便简化getElementsSingle函数 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
val rsRule = RuleAnalyzer(ruleStr) |
|
|
|
val rsRule = RuleAnalyzer(ruleStr) |
|
|
|
|
|
|
|
|
|
|
|
if( rsRule.peek() =='@' || rsRule.peek() < '!' ) rsRule.advance() // 修剪当前规则之前的"@"或者空白符 |
|
|
|
rsRule.trim() // 修剪当前规则之前的"@"或者空白符 |
|
|
|
|
|
|
|
|
|
|
|
val rs = rsRule.splitRule("@") |
|
|
|
val rs = rsRule.splitRule("@") |
|
|
|
|
|
|
|
|
|
|
@ -157,7 +156,7 @@ class AnalyzeByJSoup(doc: Any) { |
|
|
|
el.addAll(es) |
|
|
|
el.addAll(es) |
|
|
|
} |
|
|
|
} |
|
|
|
el |
|
|
|
el |
|
|
|
}else getElementsSingle(temp,ruleStr) |
|
|
|
}else ElementsSingle().getElementsSingle(temp,ruleStr) |
|
|
|
|
|
|
|
|
|
|
|
elementsList.add(el) |
|
|
|
elementsList.add(el) |
|
|
|
if (el.size > 0 && ruleAnalyzes.elementsType == "||") { |
|
|
|
if (el.size > 0 && ruleAnalyzes.elementsType == "||") { |
|
|
@ -183,6 +182,131 @@ class AnalyzeByJSoup(doc: Any) { |
|
|
|
return elements |
|
|
|
return elements |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
/** |
|
|
|
|
|
|
|
* 获取内容列表 |
|
|
|
|
|
|
|
*/ |
|
|
|
|
|
|
|
private fun getResultList(ruleStr: String): List<String>? { |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if (ruleStr.isEmpty()) return null |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
var elements = Elements() |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
elements.add(element) |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
val rule = RuleAnalyzer(ruleStr) //创建解析 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
rule.trim() //修建前置赘余符号 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
val rules = rule.splitRule("@") // 切割成列表 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
val last = rules.size - 1 |
|
|
|
|
|
|
|
for (i in 0 until last) { |
|
|
|
|
|
|
|
val es = Elements() |
|
|
|
|
|
|
|
for (elt in elements) { |
|
|
|
|
|
|
|
es.addAll(ElementsSingle().getElementsSingle(elt, rules[i])) |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
elements.clear() |
|
|
|
|
|
|
|
elements = es |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
return if (elements.isEmpty()) null else getResultLast(elements, rules[last]) |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
/** |
|
|
|
|
|
|
|
* 根据最后一个规则获取内容 |
|
|
|
|
|
|
|
*/ |
|
|
|
|
|
|
|
private fun getResultLast(elements: Elements, lastRule: String): List<String> { |
|
|
|
|
|
|
|
val textS = ArrayList<String>() |
|
|
|
|
|
|
|
try { |
|
|
|
|
|
|
|
when (lastRule) { |
|
|
|
|
|
|
|
"text" -> for (element in elements) { |
|
|
|
|
|
|
|
textS.add(element.text()) |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
"textNodes" -> for (element in elements) { |
|
|
|
|
|
|
|
val tn = arrayListOf<String>() |
|
|
|
|
|
|
|
val contentEs = element.textNodes() |
|
|
|
|
|
|
|
for (item in contentEs) { |
|
|
|
|
|
|
|
val temp = item.text().trim { it <= ' ' } |
|
|
|
|
|
|
|
if (temp.isNotEmpty()) { |
|
|
|
|
|
|
|
tn.add(temp) |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
textS.add(join("\n", tn)) |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
"ownText" -> for (element in elements) { |
|
|
|
|
|
|
|
textS.add(element.ownText()) |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
"html" -> { |
|
|
|
|
|
|
|
elements.select("script").remove() |
|
|
|
|
|
|
|
elements.select("style").remove() |
|
|
|
|
|
|
|
val html = elements.outerHtml() |
|
|
|
|
|
|
|
textS.add(html) |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
"all" -> textS.add(elements.outerHtml()) |
|
|
|
|
|
|
|
else -> for (element in elements) { |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
val url = element.attr(lastRule) |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if(url.isEmpty() || textS.contains(url)) break |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
textS.add(url) |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
} catch (e: Exception) { |
|
|
|
|
|
|
|
e.printStackTrace() |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
return textS |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
data class ElementsSingle(var split:Char = '.', |
|
|
|
|
|
|
|
var beforeRule:String = "", |
|
|
|
|
|
|
|
val indexDefault:MutableList<Int> = mutableListOf(), |
|
|
|
|
|
|
|
val indexs:MutableList<Any> = mutableListOf()){ |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
/** |
|
|
|
|
|
|
|
* 获取Elements按照一个规则 |
|
|
|
|
|
|
|
*/ |
|
|
|
|
|
|
|
fun getElementsSingle(temp: Element, rule: String): Elements { |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
var elements = Elements() |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
findIndexSet(rule) //执行索引列表处理器 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
val rules = beforeRule.split(".") |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
elements.addAll( |
|
|
|
|
|
|
|
if(beforeRule.isEmpty()) temp.children() //允许索引直接作为根元素,此时前置规则为空,效果与children相同 |
|
|
|
|
|
|
|
else when (rules[0]) { |
|
|
|
|
|
|
|
"children" -> temp.children() //允许索引直接作为根元素,此时前置规则为空,效果与children相同 |
|
|
|
|
|
|
|
"class" -> temp.getElementsByClass(rules[1]) |
|
|
|
|
|
|
|
"tag" -> temp.getElementsByTag(rules[1]) |
|
|
|
|
|
|
|
"id" -> Collector.collect(Evaluator.Id(rules[1]), temp) |
|
|
|
|
|
|
|
"text" -> temp.getElementsContainingOwnText(rules[1]) |
|
|
|
|
|
|
|
else -> temp.select(beforeRule) |
|
|
|
|
|
|
|
} ) |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
val indexSet = getIndexs(elements.size) //传入元素数量,处理负数索引及索引越界问题,生成可用索引集合。 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if(split == '!'){ //排除 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
for (pcInt in indexSet) elements[pcInt] = null |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
elements.removeAll(listOf(null)) //测试过,这样就行 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
}else if(split == '.'){ //选择 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
val es = Elements() |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
for (pcInt in indexSet) es.add(elements[pcInt]) |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
elements = es |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
return elements |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
/** |
|
|
|
/** |
|
|
|
* 1.支持阅读原有写法,':'分隔索引,!或.表示筛选方式,索引可为负数 |
|
|
|
* 1.支持阅读原有写法,':'分隔索引,!或.表示筛选方式,索引可为负数 |
|
|
|
* |
|
|
|
* |
|
|
@ -201,10 +325,8 @@ class AnalyzeByJSoup(doc: Any) { |
|
|
|
* 特殊用法 tag.div[-1:0] 可在任意地方让列表反向 |
|
|
|
* 特殊用法 tag.div[-1:0] 可在任意地方让列表反向 |
|
|
|
* |
|
|
|
* |
|
|
|
* */ |
|
|
|
* */ |
|
|
|
|
|
|
|
fun findIndexSet( rule:String ): ElementsSingle { |
|
|
|
|
|
|
|
|
|
|
|
fun findIndexSet( rule:String ): IndexSet { |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
val indexSet = IndexSet() |
|
|
|
|
|
|
|
val rus = rule.trim{ it <= ' '} |
|
|
|
val rus = rule.trim{ it <= ' '} |
|
|
|
|
|
|
|
|
|
|
|
var len = rus.length |
|
|
|
var len = rus.length |
|
|
@ -241,25 +363,26 @@ class AnalyzeByJSoup(doc: Any) { |
|
|
|
|
|
|
|
|
|
|
|
if(curInt == null) break //是jsoup选择器而非索引列表,跳出 |
|
|
|
if(curInt == null) break //是jsoup选择器而非索引列表,跳出 |
|
|
|
|
|
|
|
|
|
|
|
indexSet.indexs.add(curInt) |
|
|
|
indexs.add(curInt) |
|
|
|
} |
|
|
|
} |
|
|
|
else{ |
|
|
|
else{ |
|
|
|
|
|
|
|
|
|
|
|
//列表最后压入的是区间右端,若列表有两位则最先压入的是间隔 |
|
|
|
//列表最后压入的是区间右端,若列表有两位则最先压入的是间隔 |
|
|
|
indexSet.indexs.add( Triple(curInt, curList.last(), if(curList.size == 2) curList.first() else 1) ) |
|
|
|
indexs.add( Triple(curInt, curList.last(), if(curList.size == 2) curList.first() else 1) ) |
|
|
|
|
|
|
|
|
|
|
|
curList.clear() //重置临时列表,避免影响到下个区间的处理 |
|
|
|
curList.clear() //重置临时列表,避免影响到下个区间的处理 |
|
|
|
|
|
|
|
|
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
if(rl == '!'){ |
|
|
|
if(rl == '!'){ |
|
|
|
indexSet.split='!' |
|
|
|
split='!' |
|
|
|
do{ rl = rus[--len] } while (len > 0 && rl == ' ')//跳过所有空格 |
|
|
|
do{ rl = rus[--len] } while (len > 0 && rl == ' ')//跳过所有空格 |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
if(rl == '[') return indexSet.apply { |
|
|
|
if(rl == '[') { |
|
|
|
beforeRule = rus.substring(0, len) |
|
|
|
beforeRule = rus.substring(0, len) //遇到索引边界,返回结果 |
|
|
|
} //遇到索引边界,返回结果 |
|
|
|
return this |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
if(rl != ',') break //非索引结构,跳出 |
|
|
|
if(rl != ',') break //非索引结构,跳出 |
|
|
|
|
|
|
|
|
|
|
@ -281,11 +404,12 @@ class AnalyzeByJSoup(doc: Any) { |
|
|
|
|
|
|
|
|
|
|
|
if(rl == '!' || rl == '.' || rl == ':') { //分隔符或起始符 |
|
|
|
if(rl == '!' || rl == '.' || rl == ':') { //分隔符或起始符 |
|
|
|
|
|
|
|
|
|
|
|
indexSet.indexDefault.add(if (curMinus) -l.toInt() else l.toInt()) // 当前数字追加到列表 |
|
|
|
indexDefault.add(if (curMinus) -l.toInt() else l.toInt()) // 当前数字追加到列表 |
|
|
|
|
|
|
|
|
|
|
|
if (rl != ':') return indexSet.apply { //rl == '!' || rl == '.' |
|
|
|
if (rl != ':'){ //rl == '!' || rl == '.' |
|
|
|
split = rl |
|
|
|
split = rl |
|
|
|
beforeRule = rus.substring(0, len) |
|
|
|
beforeRule = rus.substring(0, len) |
|
|
|
|
|
|
|
return this |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
}else break //非索引结构,跳出循环 |
|
|
|
}else break //非索引结构,跳出循环 |
|
|
@ -296,142 +420,12 @@ class AnalyzeByJSoup(doc: Any) { |
|
|
|
|
|
|
|
|
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
return indexSet.apply{ |
|
|
|
|
|
|
|
split = ' ' |
|
|
|
split = ' ' |
|
|
|
beforeRule = rus } //非索引格式 |
|
|
|
beforeRule = rus |
|
|
|
} |
|
|
|
return this //非索引格式 |
|
|
|
|
|
|
|
|
|
|
|
/** |
|
|
|
|
|
|
|
* 获取Elements按照一个规则 |
|
|
|
|
|
|
|
*/ |
|
|
|
|
|
|
|
private fun getElementsSingle(temp: Element, rule: String): Elements { |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
var elements = Elements() |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
val fi = findIndexSet(rule) //执行索引列表处理器 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
val (filterType,ruleStr) = fi //获取操作类型及非索引部分的规则字串 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// val rulePc = rulePcx[0].trim { it <= ' ' }.split(">") |
|
|
|
|
|
|
|
// jsoup中,当前节点是参与选择的,tag.div 与 tag.div@tag.div 结果相同 |
|
|
|
|
|
|
|
// 此处">"效果和“@”完全相同,且容易让人误解成选择子节点,实际并不是。以后不允许这种无意义的写法 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
val rules = ruleStr.split(".") |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
elements.addAll( |
|
|
|
|
|
|
|
if(ruleStr.isEmpty()) temp.children() //允许索引直接作为根元素,此时前置规则为空,效果与children相同 |
|
|
|
|
|
|
|
else when (rules[0]) { |
|
|
|
|
|
|
|
"children" -> temp.children() //允许索引直接作为根元素,此时前置规则为空,效果与children相同 |
|
|
|
|
|
|
|
"class" -> temp.getElementsByClass(rules[1]) |
|
|
|
|
|
|
|
"tag" -> temp.getElementsByTag(rules[1]) |
|
|
|
|
|
|
|
"id" -> Collector.collect(Evaluator.Id(rules[1]), temp) |
|
|
|
|
|
|
|
"text" -> temp.getElementsContainingOwnText(rules[1]) |
|
|
|
|
|
|
|
else -> temp.select(ruleStr) |
|
|
|
|
|
|
|
} ) |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
val indexSet = fi.getIndexs(elements.size) //传入元素数量,处理负数索引及索引越界问题,生成可用索引集合。 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if(filterType == '!'){ //排除 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
for (pcInt in indexSet) elements[pcInt] = null |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
elements.removeAll(listOf(null)) //测试过,这样就行 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
}else if(filterType == '.'){ //选择 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
val es = Elements() |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
for (pcInt in indexSet) es.add(elements[pcInt]) |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
elements = es |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
return elements |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
/** |
|
|
|
|
|
|
|
* 获取内容列表 |
|
|
|
|
|
|
|
*/ |
|
|
|
|
|
|
|
private fun getResultList(ruleStr: String): List<String>? { |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if (ruleStr.isEmpty()) return null |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
var elements = Elements() |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
elements.add(element) |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
val rule = RuleAnalyzer(ruleStr) //创建解析 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
while( rule.peek() =='@' || rule.peek() < '!' ) rule.advance() // 修剪当前规则之前的"@"或者空白符 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
val rules = rule.splitRule("@") // 切割成列表 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
val last = rules.size - 1 |
|
|
|
|
|
|
|
for (i in 0 until last) { |
|
|
|
|
|
|
|
val es = Elements() |
|
|
|
|
|
|
|
for (elt in elements) { |
|
|
|
|
|
|
|
es.addAll(getElementsSingle(elt, rules[i])) |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
elements.clear() |
|
|
|
|
|
|
|
elements = es |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
return if (elements.isEmpty()) null else getResultLast(elements, rules[last]) |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
/** |
|
|
|
|
|
|
|
* 根据最后一个规则获取内容 |
|
|
|
|
|
|
|
*/ |
|
|
|
|
|
|
|
private fun getResultLast(elements: Elements, lastRule: String): List<String> { |
|
|
|
|
|
|
|
val textS = ArrayList<String>() |
|
|
|
|
|
|
|
try { |
|
|
|
|
|
|
|
when (lastRule) { |
|
|
|
|
|
|
|
"text" -> for (element in elements) { |
|
|
|
|
|
|
|
textS.add(element.text()) |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
"textNodes" -> for (element in elements) { |
|
|
|
|
|
|
|
val tn = arrayListOf<String>() |
|
|
|
|
|
|
|
val contentEs = element.textNodes() |
|
|
|
|
|
|
|
for (item in contentEs) { |
|
|
|
|
|
|
|
val temp = item.text().trim { it <= ' ' } |
|
|
|
|
|
|
|
if (temp.isNotEmpty()) { |
|
|
|
|
|
|
|
tn.add(temp) |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
textS.add(join("\n", tn)) |
|
|
|
|
|
|
|
} |
|
|
|
} |
|
|
|
"ownText" -> for (element in elements) { |
|
|
|
|
|
|
|
textS.add(element.ownText()) |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
"html" -> { |
|
|
|
|
|
|
|
elements.select("script").remove() |
|
|
|
|
|
|
|
elements.select("style").remove() |
|
|
|
|
|
|
|
val html = elements.outerHtml() |
|
|
|
|
|
|
|
textS.add(html) |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
"all" -> textS.add(elements.outerHtml()) |
|
|
|
|
|
|
|
else -> for (element in elements) { |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
val url = element.attr(lastRule) |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if(url.isEmpty() || textS.contains(url)) break |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
textS.add(url) |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
} catch (e: Exception) { |
|
|
|
|
|
|
|
e.printStackTrace() |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
return textS |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
data class IndexSet(var split:Char = '.', |
|
|
|
|
|
|
|
var beforeRule:String = "", |
|
|
|
|
|
|
|
val indexDefault:MutableList<Int> = mutableListOf(), |
|
|
|
|
|
|
|
val indexs:MutableList<Any> = mutableListOf()){ |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
fun getIndexs(len:Int): MutableSet<Int> { |
|
|
|
private fun getIndexs(len:Int): MutableSet<Int> { |
|
|
|
|
|
|
|
|
|
|
|
val indexSet = mutableSetOf<Int>() |
|
|
|
val indexSet = mutableSetOf<Int>() |
|
|
|
|
|
|
|
|
|
|
|