fengyuecanzhu
4 years ago
parent
b0e5d75e9d
commit
9b6334bab6
@ -0,0 +1,132 @@ |
|||||||
|
package xyz.fycz.myreader.webapi.crawler.read; |
||||||
|
|
||||||
|
import android.text.Html; |
||||||
|
|
||||||
|
import org.jsoup.Jsoup; |
||||||
|
import org.jsoup.nodes.Document; |
||||||
|
import org.jsoup.nodes.Element; |
||||||
|
import org.jsoup.select.Elements; |
||||||
|
|
||||||
|
import java.util.ArrayList; |
||||||
|
|
||||||
|
import xyz.fycz.myreader.entity.SearchBookBean; |
||||||
|
import xyz.fycz.myreader.enums.BookSource; |
||||||
|
import xyz.fycz.myreader.greendao.entity.Book; |
||||||
|
import xyz.fycz.myreader.greendao.entity.Chapter; |
||||||
|
import xyz.fycz.myreader.model.mulvalmap.ConcurrentMultiValueMap; |
||||||
|
import xyz.fycz.myreader.webapi.crawler.base.BookInfoCrawler; |
||||||
|
import xyz.fycz.myreader.webapi.crawler.base.ReadCrawler; |
||||||
|
|
||||||
|
|
||||||
|
public class BiJianReadCrawler implements ReadCrawler { |
||||||
|
public static final String NAME_SPACE = "http://www.bjcan.com"; |
||||||
|
public static final String NOVEL_SEARCH = "http://www.bjcan.com/home/search/index.html?keyword={key}"; |
||||||
|
public static final String CHARSET = "UTF-8"; |
||||||
|
public static final String SEARCH_CHARSET = "UTF-8"; |
||||||
|
|
||||||
|
@Override |
||||||
|
public String getSearchLink() { |
||||||
|
return NOVEL_SEARCH; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public String getCharset() { |
||||||
|
return CHARSET; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public String getNameSpace() { |
||||||
|
return NAME_SPACE; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public Boolean isPost() { |
||||||
|
return false; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public String getSearchCharset() { |
||||||
|
return SEARCH_CHARSET; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* 从html中获取章节正文 |
||||||
|
* |
||||||
|
* @param html |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
public String getContentFormHtml(String html) { |
||||||
|
Document doc = Jsoup.parse(html); |
||||||
|
Element divContent = doc.getElementsByClass("read-content").first(); |
||||||
|
if (divContent != null) { |
||||||
|
String content = Html.fromHtml(divContent.html()).toString(); |
||||||
|
char c = 160; |
||||||
|
String spaec = "" + c; |
||||||
|
content = content.replace(spaec, " "); |
||||||
|
return content; |
||||||
|
} else { |
||||||
|
return ""; |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* 从html中获取章节列表 |
||||||
|
* |
||||||
|
* @param html |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
public ArrayList<Chapter> getChaptersFromHtml(String html) { |
||||||
|
ArrayList<Chapter> chapters = new ArrayList<>(); |
||||||
|
Document doc = Jsoup.parse(html); |
||||||
|
Element divList = doc.getElementsByClass("attentions").get(1); |
||||||
|
Elements elementsByTag = divList.getElementsByTag("a"); |
||||||
|
for (int i = 0; i < elementsByTag.size(); i++) { |
||||||
|
Element a = elementsByTag.get(i); |
||||||
|
String title = a.text(); |
||||||
|
String url = a.attr("href"); |
||||||
|
Chapter chapter = new Chapter(); |
||||||
|
chapter.setNumber(i); |
||||||
|
chapter.setTitle(title); |
||||||
|
chapter.setUrl(url); |
||||||
|
chapters.add(chapter); |
||||||
|
} |
||||||
|
return chapters; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* 从搜索html中得到书列表 |
||||||
|
*<li> |
||||||
|
* <a class="pic" href="http://www.bjcan.com/book/74544.html" target="_blank"><img class="lazy" src="http://www.bjcan.com/uploads/novel/20200907/b38cea14e4a3de1e876395e378c1e544.jpeg" alt="大主宰：灵玖"></a> |
||||||
|
* <h5 class="tit"><a href="http://www.bjcan.com/book/74544.html" target="_blank">大主宰：灵玖</a></h5> |
||||||
|
* <p class="info">作者：<span>霞露</span><span>分类：其他</span><i class="serial">连载中</i></p> |
||||||
|
* <p class="intro">简介：她，身负系统，莫名来到了大主宰的时空，成为了聚灵族的最后族人。在她还对周围的情况一片模糊的时候，她遇到了林静这个小恶魔。拜林静所赐，她还遇到了武祖，成为了武柤的小徒弟。参与了灵路，遇到了牧尘和洛璃，她默默的在心中问着自己那一直在划水的系统：“你的活来了，说吧，我该干什么？”</p> |
||||||
|
* <a class="view" href="http://www.bjcan.com/book/74544.html" target="_blank">小说详情</a> |
||||||
|
*</li> |
||||||
|
*/ |
||||||
|
public ConcurrentMultiValueMap<SearchBookBean, Book> getBooksFromSearchHtml(String html) { |
||||||
|
ConcurrentMultiValueMap<SearchBookBean, Book> books = new ConcurrentMultiValueMap<>(); |
||||||
|
Document doc = Jsoup.parse(html); |
||||||
|
Elements divs = doc.getElementsByClass("book-list"); |
||||||
|
Element div = divs.get(0); |
||||||
|
Elements elementsByTag = div.getElementsByTag("li"); |
||||||
|
for (int i = 0; i < elementsByTag.size(); i++) { |
||||||
|
Element element = elementsByTag.get(i); |
||||||
|
Elements as = element.getElementsByTag("a"); |
||||||
|
Elements ps = element.getElementsByTag("p"); |
||||||
|
Book book = new Book(); |
||||||
|
book.setImgUrl(element.getElementsByTag("img").attr("src")); |
||||||
|
book.setName(as.get(1).text()); |
||||||
|
Elements spans = ps.get(0).getElementsByTag("span"); |
||||||
|
book.setAuthor(spans.get(0).text()); |
||||||
|
book.setType(spans.get(1).text().replace("分类：", "")); |
||||||
|
book.setChapterUrl(as.get(2).attr("href")); |
||||||
|
book.setDesc(ps.get(1).text()); |
||||||
|
book.setNewestChapterTitle(""); |
||||||
|
book.setSource(BookSource.bijian.toString()); |
||||||
|
SearchBookBean sbb = new SearchBookBean(book.getName(), book.getAuthor()); |
||||||
|
books.add(sbb, book); |
||||||
|
} |
||||||
|
return books; |
||||||
|
} |
||||||
|
|
||||||
|
} |
||||||
@ -0,0 +1,135 @@ |
|||||||
|
package xyz.fycz.myreader.webapi.crawler.read; |
||||||
|
|
||||||
|
import android.text.Html; |
||||||
|
|
||||||
|
import org.jsoup.Jsoup; |
||||||
|
import org.jsoup.nodes.Document; |
||||||
|
import org.jsoup.nodes.Element; |
||||||
|
import org.jsoup.select.Elements; |
||||||
|
|
||||||
|
import java.util.ArrayList; |
||||||
|
|
||||||
|
import xyz.fycz.myreader.entity.SearchBookBean; |
||||||
|
import xyz.fycz.myreader.enums.BookSource; |
||||||
|
import xyz.fycz.myreader.greendao.entity.Book; |
||||||
|
import xyz.fycz.myreader.greendao.entity.Chapter; |
||||||
|
import xyz.fycz.myreader.model.mulvalmap.ConcurrentMultiValueMap; |
||||||
|
import xyz.fycz.myreader.webapi.crawler.base.ReadCrawler; |
||||||
|
|
||||||
|
|
||||||
|
public class WoLongReadCrawler implements ReadCrawler { |
||||||
|
public static final String NAME_SPACE = "http://www.paper027.com"; |
||||||
|
public static final String NOVEL_SEARCH = "http://www.paper027.com/search.html?keyword={key}"; |
||||||
|
public static final String CHARSET = "UTF-8"; |
||||||
|
public static final String SEARCH_CHARSET = "UTF-8"; |
||||||
|
|
||||||
|
@Override |
||||||
|
public String getSearchLink() { |
||||||
|
return NOVEL_SEARCH; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public String getCharset() { |
||||||
|
return CHARSET; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public String getNameSpace() { |
||||||
|
return NAME_SPACE; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public Boolean isPost() { |
||||||
|
return false; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public String getSearchCharset() { |
||||||
|
return SEARCH_CHARSET; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* 从html中获取章节正文 |
||||||
|
* |
||||||
|
* @param html |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
public String getContentFormHtml(String html) { |
||||||
|
Document doc = Jsoup.parse(html); |
||||||
|
Element divContent = doc.getElementById("contentsource"); |
||||||
|
if (divContent != null) { |
||||||
|
String content = Html.fromHtml(divContent.html()).toString(); |
||||||
|
char c = 160; |
||||||
|
String spaec = "" + c; |
||||||
|
content = content.replace(spaec, " "); |
||||||
|
return content; |
||||||
|
} else { |
||||||
|
return ""; |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* 从html中获取章节列表 |
||||||
|
* |
||||||
|
* @param html |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
public ArrayList<Chapter> getChaptersFromHtml(String html) { |
||||||
|
ArrayList<Chapter> chapters = new ArrayList<>(); |
||||||
|
Document doc = Jsoup.parse(html); |
||||||
|
Element divList = doc.getElementsByClass("chapters").get(0); |
||||||
|
Elements elementsByTag = divList.getElementsByTag("a"); |
||||||
|
int j = 0; |
||||||
|
for (int i = elementsByTag.size() - 1; i >= 0; i--) { |
||||||
|
Element a = elementsByTag.get(i); |
||||||
|
String title = a.text(); |
||||||
|
String url = a.attr("href"); |
||||||
|
Chapter chapter = new Chapter(); |
||||||
|
chapter.setNumber(j++); |
||||||
|
chapter.setTitle(title); |
||||||
|
chapter.setUrl(url); |
||||||
|
chapters.add(chapter); |
||||||
|
} |
||||||
|
return chapters; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* 从搜索html中得到书列表 |
||||||
|
<div> |
||||||
|
<a href="http://www.paper027.com/novel/75432.html" target="_blank"><img class="img-rounded" src="http://www.paper027.com/uploads/novel/20190802/9883298e2e72ecfa53fabf0ef2e03e21.jpg"/></a> |
||||||
|
<h2><a href="http://www.paper027.com/novel/75432.html" target="_blank">大主宰之混子日常</a></h2> |
||||||
|
<p class="text-muted"><span>錯過过错</span> <small>2019-08-06 19:14</small></p> |
||||||
|
</div> |
||||||
|
<div class="clearfix searchresult-info"> |
||||||
|
<p><a href="http://www.paper027.com/novel/75432.html" target="_blank">作者很懒，什么也没有留下。...</a></p> |
||||||
|
<ul class="list-inline text-muted"> |
||||||
|
<li>11635人看过</li> |
||||||
|
<li>标签：</li> |
||||||
|
<li><a class="text-warning">同人衍生</a></li> |
||||||
|
</ul> |
||||||
|
</div> |
||||||
|
*/ |
||||||
|
public ConcurrentMultiValueMap<SearchBookBean, Book> getBooksFromSearchHtml(String html) { |
||||||
|
ConcurrentMultiValueMap<SearchBookBean, Book> books = new ConcurrentMultiValueMap<>(); |
||||||
|
Document doc = Jsoup.parse(html); |
||||||
|
Elements elementsByTag = doc.getElementsByClass("searchresult"); |
||||||
|
for (int i = 0; i < elementsByTag.size(); i++) { |
||||||
|
Element element = elementsByTag.get(i); |
||||||
|
Elements as = element.getElementsByTag("a"); |
||||||
|
Elements ps = element.getElementsByTag("p"); |
||||||
|
Book book = new Book(); |
||||||
|
book.setImgUrl(element.getElementsByTag("img").attr("src")); |
||||||
|
book.setName(as.get(1).text()); |
||||||
|
book.setAuthor(ps.get(0).getElementsByTag("span").get(0).text()); |
||||||
|
book.setType(as.get(3).text()); |
||||||
|
book.setChapterUrl(as.get(1).attr("href").replace("novel", "home/chapter/lists/id")); |
||||||
|
book.setDesc(as.get(2).text()); |
||||||
|
book.setNewestChapterTitle(""); |
||||||
|
book.setSource(BookSource.wolong.toString()); |
||||||
|
SearchBookBean sbb = new SearchBookBean(book.getName(), book.getAuthor()); |
||||||
|
books.add(sbb, book); |
||||||
|
} |
||||||
|
return books; |
||||||
|
} |
||||||
|
|
||||||
|
} |
||||||
@ -0,0 +1,165 @@ |
|||||||
|
package xyz.fycz.myreader.webapi.crawler.read; |
||||||
|
|
||||||
|
import android.text.Html; |
||||||
|
|
||||||
|
import org.jsoup.Jsoup; |
||||||
|
import org.jsoup.nodes.Document; |
||||||
|
import org.jsoup.nodes.Element; |
||||||
|
import org.jsoup.select.Elements; |
||||||
|
|
||||||
|
import java.util.ArrayList; |
||||||
|
|
||||||
|
import xyz.fycz.myreader.entity.SearchBookBean; |
||||||
|
import xyz.fycz.myreader.enums.BookSource; |
||||||
|
import xyz.fycz.myreader.greendao.entity.Book; |
||||||
|
import xyz.fycz.myreader.greendao.entity.Chapter; |
||||||
|
import xyz.fycz.myreader.model.mulvalmap.ConcurrentMultiValueMap; |
||||||
|
import xyz.fycz.myreader.webapi.crawler.base.ReadCrawler; |
||||||
|
|
||||||
|
|
||||||
|
public class YanQingLouReadCrawler implements ReadCrawler { |
||||||
|
public static final String NAME_SPACE = "http://www.yanqinglou.com"; |
||||||
|
public static final String NOVEL_SEARCH = "http://www.yanqinglou.com/Home/Search,action=search&q={key}"; |
||||||
|
public static final String CHARSET = "UTF-8"; |
||||||
|
public static final String SEARCH_CHARSET = "UTF-8"; |
||||||
|
|
||||||
|
@Override |
||||||
|
public String getSearchLink() { |
||||||
|
return NOVEL_SEARCH; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public String getCharset() { |
||||||
|
return CHARSET; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public String getNameSpace() { |
||||||
|
return NAME_SPACE; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public Boolean isPost() { |
||||||
|
return true; |
||||||
|
} |
||||||
|
|
||||||
|
@Override |
||||||
|
public String getSearchCharset() { |
||||||
|
return SEARCH_CHARSET; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* 从html中获取章节正文 |
||||||
|
* |
||||||
|
* @param html |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
public String getContentFormHtml(String html) { |
||||||
|
Document doc = Jsoup.parse(html); |
||||||
|
Element divContent = doc.getElementById("content"); |
||||||
|
if (divContent != null) { |
||||||
|
String content = Html.fromHtml(divContent.html()).toString(); |
||||||
|
char c = 160; |
||||||
|
String spaec = "" + c; |
||||||
|
content = content.replace(spaec, " "); |
||||||
|
return content; |
||||||
|
} else { |
||||||
|
return ""; |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* 从html中获取章节列表 |
||||||
|
* |
||||||
|
* @param html |
||||||
|
* @return |
||||||
|
*/ |
||||||
|
public ArrayList<Chapter> getChaptersFromHtml(String html) { |
||||||
|
ArrayList<Chapter> chapters = new ArrayList<>(); |
||||||
|
Document doc = Jsoup.parse(html); |
||||||
|
Element divList = doc.getElementsByClass("fulllistall").first(); |
||||||
|
Elements elementsByTag = divList.getElementsByTag("a"); |
||||||
|
for (int i = 0; i < elementsByTag.size(); i++) { |
||||||
|
Element a = elementsByTag.get(i); |
||||||
|
String title = a.text(); |
||||||
|
String url = a.attr("href"); |
||||||
|
Chapter chapter = new Chapter(); |
||||||
|
chapter.setNumber(i); |
||||||
|
chapter.setTitle(title); |
||||||
|
chapter.setUrl(NAME_SPACE + url); |
||||||
|
chapters.add(chapter); |
||||||
|
} |
||||||
|
return chapters; |
||||||
|
} |
||||||
|
|
||||||
|
/** |
||||||
|
* 从搜索html中得到书列表 |
||||||
|
<div class="bookbox"> |
||||||
|
<div class="p10"><span class="num"> <a title="斗罗之大主宰" href="/hunlian/459337/"><img layout="fixed" width="90" height="120" src="https://www.biquduo.com/files/article/image/64/64075/64075s.jpg" alt="斗罗之大主宰" /></a> </span> |
||||||
|
<div class="bookinfo"> |
||||||
|
<h4 class="bookname"><a title="斗罗之大主宰" href="/hunlian/459337/">斗罗之大主宰</a></h4> |
||||||
|
<div class="author">作者：<a href="/writter/%E4%B8%8A%E5%BC%A6666.html" target="_blank" title="上弦666的作品大全">上弦666</a></div> |
||||||
|
<div class="author">分类：婚恋</div> |
||||||
|
<div class="cat"><span>更新到：</span><a href="/hunlian/459337/">最近更新>></a></div> |
||||||
|
<div class="update"><span>简介：</span>叶辰，武魂，昊天锤的变异，大须弥锤！昊天锤本就是大陆第一器武魂，而武魂是大须弥锤的叶辰，会有多强？当</div> |
||||||
|
</div> |
||||||
|
<div class="delbutton"> <a class="del_but" title="斗罗之大主宰" href="/hunlian/459337/">阅读</a></div> |
||||||
|
</div> |
||||||
|
</div> |
||||||
|
*/ |
||||||
|
public ConcurrentMultiValueMap<SearchBookBean, Book> getBooksFromSearchHtml(String html) { |
||||||
|
ConcurrentMultiValueMap<SearchBookBean, Book> books = new ConcurrentMultiValueMap<>(); |
||||||
|
Document doc = Jsoup.parse(html); |
||||||
|
String bookName = doc.select("meta[property=og:novel:book_name]").attr("content"); |
||||||
|
if ("".equals(bookName)) { |
||||||
|
Element div = doc.getElementsByClass("keywords").first(); |
||||||
|
Elements divs = div.getElementsByClass("bookbox"); |
||||||
|
for (Element divBook : divs) { |
||||||
|
Elements as = divBook.getElementsByTag("a"); |
||||||
|
Book book = new Book(); |
||||||
|
book.setName(as.get(1).text()); |
||||||
|
book.setAuthor(as.get(2).text()); |
||||||
|
book.setType(divBook.getElementsByClass("author").get(1).text().replace("分类：", "")); |
||||||
|
book.setNewestChapterTitle(as.get(3).text().replace("最近更新>>", "")); |
||||||
|
book.setDesc(divBook.getElementsByClass("update").first().text().replace("简介：", "")); |
||||||
|
book.setImgUrl(divBook.getElementsByTag("img").attr("src")); |
||||||
|
book.setChapterUrl(NAME_SPACE + as.get(0).attr("href")); |
||||||
|
book.setSource(BookSource.yanqinglou.toString()); |
||||||
|
SearchBookBean sbb = new SearchBookBean(book.getName(), book.getAuthor()); |
||||||
|
books.add(sbb, book); |
||||||
|
} |
||||||
|
}else { |
||||||
|
Book book = new Book(); |
||||||
|
getBookInfo(doc, book); |
||||||
|
SearchBookBean sbb = new SearchBookBean(book.getName(), book.getAuthor()); |
||||||
|
books.add(sbb, book); |
||||||
|
} |
||||||
|
return books; |
||||||
|
} |
||||||
|
|
||||||
|
private void getBookInfo(Document doc, Book book) { |
||||||
|
String name = doc.select("meta[property=og:novel:book_name]").attr("content"); |
||||||
|
book.setName(name); |
||||||
|
|
||||||
|
String author = doc.select("meta[property=og:novel:author]").attr("content"); |
||||||
|
book.setAuthor(author); |
||||||
|
|
||||||
|
String type = doc.select("meta[property=og:novel:category]").attr("content"); |
||||||
|
book.setType(type); |
||||||
|
|
||||||
|
String newestTitle = doc.select("meta[property=og:novel:latest_chapter_name]").attr("content"); |
||||||
|
book.setNewestChapterTitle(newestTitle); |
||||||
|
|
||||||
|
String desc = doc.select("meta[property=og:description]").attr("content"); |
||||||
|
book.setDesc(desc); |
||||||
|
|
||||||
|
String img = doc.select("meta[property=og:image]").attr("content"); |
||||||
|
book.setImgUrl(NAME_SPACE + img); |
||||||
|
|
||||||
|
String url = doc.select("meta[property=og:novel:read_url]").attr("content"); |
||||||
|
book.setChapterUrl(url); |
||||||
|
book.setSource(BookSource.yanqinglou.toString()); |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
} |
||||||
@ -1,2 +1,2 @@ |
|||||||
#Mon Jan 25 21:29:58 CST 2021 |
#Mon Feb 01 19:33:27 CST 2021 |
||||||
VERSION_CODE=180 |
VERSION_CODE=181 |
||||||
|
|||||||
Loading…
Reference in new issue