|
|
@ -46,7 +46,7 @@ class AnalyzeUrl( |
|
|
|
companion object { |
|
|
|
companion object { |
|
|
|
val paramPattern: Pattern = Pattern.compile("\\s*,\\s*(?=\\{)") |
|
|
|
val paramPattern: Pattern = Pattern.compile("\\s*,\\s*(?=\\{)") |
|
|
|
private val pagePattern = Pattern.compile("<(.*?)>") |
|
|
|
private val pagePattern = Pattern.compile("<(.*?)>") |
|
|
|
private val accessTime = hashMapOf<String, FetchRecord>() |
|
|
|
private val concurrentRecordMap = hashMapOf<String, ConcurrentRecord>() |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
var ruleUrl = "" |
|
|
|
var ruleUrl = "" |
|
|
@ -277,31 +277,34 @@ class AnalyzeUrl( |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
/** |
|
|
|
/** |
|
|
|
* 并发判断 |
|
|
|
* 开始访问,并发判断 |
|
|
|
*/ |
|
|
|
*/ |
|
|
|
private fun judgmentConcurrent() { |
|
|
|
private fun fetchStart(): ConcurrentRecord? { |
|
|
|
source ?: return |
|
|
|
source ?: return null |
|
|
|
val concurrentRate = source.concurrentRate |
|
|
|
val concurrentRate = source.concurrentRate |
|
|
|
if (concurrentRate.isNullOrEmpty()) { |
|
|
|
if (concurrentRate.isNullOrEmpty()) { |
|
|
|
return |
|
|
|
return null |
|
|
|
} |
|
|
|
} |
|
|
|
var fetchRecord = accessTime[source.getKey()] |
|
|
|
val rateIndex = concurrentRate.indexOf("/") |
|
|
|
|
|
|
|
var fetchRecord = concurrentRecordMap[source.getKey()] |
|
|
|
if (fetchRecord == null) { |
|
|
|
if (fetchRecord == null) { |
|
|
|
fetchRecord = FetchRecord(System.currentTimeMillis(), 1) |
|
|
|
fetchRecord = ConcurrentRecord(rateIndex > 0, System.currentTimeMillis(), 1) |
|
|
|
accessTime[source.getKey()] = fetchRecord |
|
|
|
concurrentRecordMap[source.getKey()] = fetchRecord |
|
|
|
return |
|
|
|
return fetchRecord |
|
|
|
} |
|
|
|
} |
|
|
|
val waitTime = synchronized(fetchRecord) { |
|
|
|
val waitTime: Int = synchronized(fetchRecord) { |
|
|
|
try { |
|
|
|
try { |
|
|
|
val rateIndex = concurrentRate.indexOf("/") |
|
|
|
|
|
|
|
if (rateIndex == -1) { |
|
|
|
if (rateIndex == -1) { |
|
|
|
|
|
|
|
if (fetchRecord.frequency > 0) { |
|
|
|
|
|
|
|
return@synchronized concurrentRate.toInt() |
|
|
|
|
|
|
|
} |
|
|
|
val nextTime = fetchRecord.time + concurrentRate.toInt() |
|
|
|
val nextTime = fetchRecord.time + concurrentRate.toInt() |
|
|
|
if (System.currentTimeMillis() >= nextTime) { |
|
|
|
if (System.currentTimeMillis() >= nextTime) { |
|
|
|
fetchRecord.time = System.currentTimeMillis() |
|
|
|
fetchRecord.time = System.currentTimeMillis() |
|
|
|
fetchRecord.frequency = 1 |
|
|
|
fetchRecord.frequency = 1 |
|
|
|
return@synchronized 0 |
|
|
|
return@synchronized 0 |
|
|
|
} |
|
|
|
} |
|
|
|
return@synchronized nextTime - System.currentTimeMillis() |
|
|
|
return@synchronized (nextTime - System.currentTimeMillis()).toInt() |
|
|
|
} else { |
|
|
|
} else { |
|
|
|
val sj = concurrentRate.substring(rateIndex + 1) |
|
|
|
val sj = concurrentRate.substring(rateIndex + 1) |
|
|
|
val nextTime = fetchRecord.time + sj.toInt() |
|
|
|
val nextTime = fetchRecord.time + sj.toInt() |
|
|
@ -312,7 +315,7 @@ class AnalyzeUrl( |
|
|
|
} |
|
|
|
} |
|
|
|
val cs = concurrentRate.substring(0, rateIndex) |
|
|
|
val cs = concurrentRate.substring(0, rateIndex) |
|
|
|
if (fetchRecord.frequency > cs.toInt()) { |
|
|
|
if (fetchRecord.frequency > cs.toInt()) { |
|
|
|
return@synchronized nextTime - System.currentTimeMillis() |
|
|
|
return@synchronized (nextTime - System.currentTimeMillis()).toInt() |
|
|
|
} else { |
|
|
|
} else { |
|
|
|
fetchRecord.frequency = fetchRecord.frequency + 1 |
|
|
|
fetchRecord.frequency = fetchRecord.frequency + 1 |
|
|
|
return@synchronized 0 |
|
|
|
return@synchronized 0 |
|
|
@ -325,6 +328,18 @@ class AnalyzeUrl( |
|
|
|
if (waitTime > 0) { |
|
|
|
if (waitTime > 0) { |
|
|
|
throw ConcurrentException("根据并发率还需等待${waitTime}毫秒才可以访问", waitTime = waitTime) |
|
|
|
throw ConcurrentException("根据并发率还需等待${waitTime}毫秒才可以访问", waitTime = waitTime) |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
return fetchRecord |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
/** |
|
|
|
|
|
|
|
* 访问结束 |
|
|
|
|
|
|
|
*/ |
|
|
|
|
|
|
|
private fun fetchEnd(concurrentRecord: ConcurrentRecord?) { |
|
|
|
|
|
|
|
if (concurrentRecord != null && !concurrentRecord.concurrent) { |
|
|
|
|
|
|
|
synchronized(concurrentRecord) { |
|
|
|
|
|
|
|
concurrentRecord.frequency = concurrentRecord.frequency - 1 |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
/** |
|
|
|
/** |
|
|
@ -338,10 +353,11 @@ class AnalyzeUrl( |
|
|
|
if (type != null) { |
|
|
|
if (type != null) { |
|
|
|
return StrResponse(url, StringUtils.byteToHexString(getByteArrayAwait())) |
|
|
|
return StrResponse(url, StringUtils.byteToHexString(getByteArrayAwait())) |
|
|
|
} |
|
|
|
} |
|
|
|
judgmentConcurrent() |
|
|
|
val concurrentRecord = fetchStart() |
|
|
|
setCookie(source?.getKey()) |
|
|
|
setCookie(source?.getKey()) |
|
|
|
|
|
|
|
val strResponse: StrResponse |
|
|
|
if (this.useWebView && useWebView) { |
|
|
|
if (this.useWebView && useWebView) { |
|
|
|
return when (method) { |
|
|
|
strResponse = when (method) { |
|
|
|
RequestMethod.POST -> { |
|
|
|
RequestMethod.POST -> { |
|
|
|
val body = getProxyClient(proxy).newCallStrResponse(retry) { |
|
|
|
val body = getProxyClient(proxy).newCallStrResponse(retry) { |
|
|
|
addHeaders(headerMap) |
|
|
|
addHeaders(headerMap) |
|
|
@ -369,21 +385,24 @@ class AnalyzeUrl( |
|
|
|
headerMap = headerMap |
|
|
|
headerMap = headerMap |
|
|
|
).getStrResponse() |
|
|
|
).getStrResponse() |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
} else { |
|
|
|
return getProxyClient(proxy).newCallStrResponse(retry) { |
|
|
|
strResponse = getProxyClient(proxy).newCallStrResponse(retry) { |
|
|
|
addHeaders(headerMap) |
|
|
|
addHeaders(headerMap) |
|
|
|
when (method) { |
|
|
|
when (method) { |
|
|
|
RequestMethod.POST -> { |
|
|
|
RequestMethod.POST -> { |
|
|
|
url(urlNoQuery) |
|
|
|
url(urlNoQuery) |
|
|
|
if (fieldMap.isNotEmpty() || body.isNullOrBlank()) { |
|
|
|
if (fieldMap.isNotEmpty() || body.isNullOrBlank()) { |
|
|
|
postForm(fieldMap, true) |
|
|
|
postForm(fieldMap, true) |
|
|
|
} else { |
|
|
|
} else { |
|
|
|
postJson(body) |
|
|
|
postJson(body) |
|
|
|
|
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
else -> get(urlNoQuery, fieldMap, true) |
|
|
|
} |
|
|
|
} |
|
|
|
else -> get(urlNoQuery, fieldMap, true) |
|
|
|
|
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
fetchEnd(concurrentRecord) |
|
|
|
|
|
|
|
return strResponse |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
@JvmOverloads |
|
|
|
@JvmOverloads |
|
|
@ -401,10 +420,10 @@ class AnalyzeUrl( |
|
|
|
* 访问网站,返回Response |
|
|
|
* 访问网站,返回Response |
|
|
|
*/ |
|
|
|
*/ |
|
|
|
suspend fun getResponseAwait(): Response { |
|
|
|
suspend fun getResponseAwait(): Response { |
|
|
|
judgmentConcurrent() |
|
|
|
val concurrentRecord = fetchStart() |
|
|
|
setCookie(source?.getKey()) |
|
|
|
setCookie(source?.getKey()) |
|
|
|
@Suppress("BlockingMethodInNonBlockingContext") |
|
|
|
@Suppress("BlockingMethodInNonBlockingContext") |
|
|
|
return getProxyClient(proxy).newCallResponse(retry) { |
|
|
|
val response = getProxyClient(proxy).newCallResponse(retry) { |
|
|
|
addHeaders(headerMap) |
|
|
|
addHeaders(headerMap) |
|
|
|
when (method) { |
|
|
|
when (method) { |
|
|
|
RequestMethod.POST -> { |
|
|
|
RequestMethod.POST -> { |
|
|
@ -418,6 +437,8 @@ class AnalyzeUrl( |
|
|
|
else -> get(urlNoQuery, fieldMap, true) |
|
|
|
else -> get(urlNoQuery, fieldMap, true) |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
fetchEnd(concurrentRecord) |
|
|
|
|
|
|
|
return response |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
fun getResponse(): Response { |
|
|
|
fun getResponse(): Response { |
|
|
@ -430,10 +451,10 @@ class AnalyzeUrl( |
|
|
|
* 访问网站,返回ByteArray |
|
|
|
* 访问网站,返回ByteArray |
|
|
|
*/ |
|
|
|
*/ |
|
|
|
suspend fun getByteArrayAwait(): ByteArray { |
|
|
|
suspend fun getByteArrayAwait(): ByteArray { |
|
|
|
judgmentConcurrent() |
|
|
|
val concurrentRecord = fetchStart() |
|
|
|
setCookie(source?.getKey()) |
|
|
|
setCookie(source?.getKey()) |
|
|
|
@Suppress("BlockingMethodInNonBlockingContext") |
|
|
|
@Suppress("BlockingMethodInNonBlockingContext") |
|
|
|
return getProxyClient(proxy).newCallResponseBody(retry) { |
|
|
|
val byteArray = getProxyClient(proxy).newCallResponseBody(retry) { |
|
|
|
addHeaders(headerMap) |
|
|
|
addHeaders(headerMap) |
|
|
|
when (method) { |
|
|
|
when (method) { |
|
|
|
RequestMethod.POST -> { |
|
|
|
RequestMethod.POST -> { |
|
|
@ -447,6 +468,8 @@ class AnalyzeUrl( |
|
|
|
else -> get(urlNoQuery, fieldMap, true) |
|
|
|
else -> get(urlNoQuery, fieldMap, true) |
|
|
|
} |
|
|
|
} |
|
|
|
}.bytes() |
|
|
|
}.bytes() |
|
|
|
|
|
|
|
fetchEnd(concurrentRecord) |
|
|
|
|
|
|
|
return byteArray |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
fun getByteArray(): ByteArray { |
|
|
|
fun getByteArray(): ByteArray { |
|
|
@ -522,7 +545,8 @@ class AnalyzeUrl( |
|
|
|
val webJs: String?, |
|
|
|
val webJs: String?, |
|
|
|
) |
|
|
|
) |
|
|
|
|
|
|
|
|
|
|
|
data class FetchRecord( |
|
|
|
data class ConcurrentRecord( |
|
|
|
|
|
|
|
val concurrent: Boolean, |
|
|
|
var time: Long, |
|
|
|
var time: Long, |
|
|
|
var frequency: Int |
|
|
|
var frequency: Int |
|
|
|
) |
|
|
|
) |
|
|
|