fengyuecanzhu
4 years ago
parent
b0e5d75e9d
commit
9b6334bab6
@ -0,0 +1,132 @@ |
||||
package xyz.fycz.myreader.webapi.crawler.read; |
||||
|
||||
import android.text.Html; |
||||
|
||||
import org.jsoup.Jsoup; |
||||
import org.jsoup.nodes.Document; |
||||
import org.jsoup.nodes.Element; |
||||
import org.jsoup.select.Elements; |
||||
|
||||
import java.util.ArrayList; |
||||
|
||||
import xyz.fycz.myreader.entity.SearchBookBean; |
||||
import xyz.fycz.myreader.enums.BookSource; |
||||
import xyz.fycz.myreader.greendao.entity.Book; |
||||
import xyz.fycz.myreader.greendao.entity.Chapter; |
||||
import xyz.fycz.myreader.model.mulvalmap.ConcurrentMultiValueMap; |
||||
import xyz.fycz.myreader.webapi.crawler.base.BookInfoCrawler; |
||||
import xyz.fycz.myreader.webapi.crawler.base.ReadCrawler; |
||||
|
||||
|
||||
public class BiJianReadCrawler implements ReadCrawler { |
||||
public static final String NAME_SPACE = "http://www.bjcan.com"; |
||||
public static final String NOVEL_SEARCH = "http://www.bjcan.com/home/search/index.html?keyword={key}"; |
||||
public static final String CHARSET = "UTF-8"; |
||||
public static final String SEARCH_CHARSET = "UTF-8"; |
||||
|
||||
@Override |
||||
public String getSearchLink() { |
||||
return NOVEL_SEARCH; |
||||
} |
||||
|
||||
@Override |
||||
public String getCharset() { |
||||
return CHARSET; |
||||
} |
||||
|
||||
@Override |
||||
public String getNameSpace() { |
||||
return NAME_SPACE; |
||||
} |
||||
|
||||
@Override |
||||
public Boolean isPost() { |
||||
return false; |
||||
} |
||||
|
||||
@Override |
||||
public String getSearchCharset() { |
||||
return SEARCH_CHARSET; |
||||
} |
||||
|
||||
/** |
||||
* 从html中获取章节正文 |
||||
* |
||||
* @param html |
||||
* @return |
||||
*/ |
||||
public String getContentFormHtml(String html) { |
||||
Document doc = Jsoup.parse(html); |
||||
Element divContent = doc.getElementsByClass("read-content").first(); |
||||
if (divContent != null) { |
||||
String content = Html.fromHtml(divContent.html()).toString(); |
||||
char c = 160; |
||||
String spaec = "" + c; |
||||
content = content.replace(spaec, " "); |
||||
return content; |
||||
} else { |
||||
return ""; |
||||
} |
||||
} |
||||
|
||||
/** |
||||
* 从html中获取章节列表 |
||||
* |
||||
* @param html |
||||
* @return |
||||
*/ |
||||
public ArrayList<Chapter> getChaptersFromHtml(String html) { |
||||
ArrayList<Chapter> chapters = new ArrayList<>(); |
||||
Document doc = Jsoup.parse(html); |
||||
Element divList = doc.getElementsByClass("attentions").get(1); |
||||
Elements elementsByTag = divList.getElementsByTag("a"); |
||||
for (int i = 0; i < elementsByTag.size(); i++) { |
||||
Element a = elementsByTag.get(i); |
||||
String title = a.text(); |
||||
String url = a.attr("href"); |
||||
Chapter chapter = new Chapter(); |
||||
chapter.setNumber(i); |
||||
chapter.setTitle(title); |
||||
chapter.setUrl(url); |
||||
chapters.add(chapter); |
||||
} |
||||
return chapters; |
||||
} |
||||
|
||||
/** |
||||
* 从搜索html中得到书列表 |
||||
*<li> |
||||
* <a class="pic" href="http://www.bjcan.com/book/74544.html" target="_blank"><img class="lazy" src="http://www.bjcan.com/uploads/novel/20200907/b38cea14e4a3de1e876395e378c1e544.jpeg" alt="大主宰：灵玖"></a> |
||||
* <h5 class="tit"><a href="http://www.bjcan.com/book/74544.html" target="_blank">大主宰：灵玖</a></h5> |
||||
* <p class="info">作者：<span>霞露</span><span>分类：其他</span><i class="serial">连载中</i></p> |
||||
* <p class="intro">简介：她，身负系统，莫名来到了大主宰的时空，成为了聚灵族的最后族人。在她还对周围的情况一片模糊的时候，她遇到了林静这个小恶魔。拜林静所赐，她还遇到了武祖，成为了武柤的小徒弟。参与了灵路，遇到了牧尘和洛璃，她默默的在心中问着自己那一直在划水的系统：“你的活来了，说吧，我该干什么？”</p> |
||||
* <a class="view" href="http://www.bjcan.com/book/74544.html" target="_blank">小说详情</a> |
||||
*</li> |
||||
*/ |
||||
public ConcurrentMultiValueMap<SearchBookBean, Book> getBooksFromSearchHtml(String html) { |
||||
ConcurrentMultiValueMap<SearchBookBean, Book> books = new ConcurrentMultiValueMap<>(); |
||||
Document doc = Jsoup.parse(html); |
||||
Elements divs = doc.getElementsByClass("book-list"); |
||||
Element div = divs.get(0); |
||||
Elements elementsByTag = div.getElementsByTag("li"); |
||||
for (int i = 0; i < elementsByTag.size(); i++) { |
||||
Element element = elementsByTag.get(i); |
||||
Elements as = element.getElementsByTag("a"); |
||||
Elements ps = element.getElementsByTag("p"); |
||||
Book book = new Book(); |
||||
book.setImgUrl(element.getElementsByTag("img").attr("src")); |
||||
book.setName(as.get(1).text()); |
||||
Elements spans = ps.get(0).getElementsByTag("span"); |
||||
book.setAuthor(spans.get(0).text()); |
||||
book.setType(spans.get(1).text().replace("分类：", "")); |
||||
book.setChapterUrl(as.get(2).attr("href")); |
||||
book.setDesc(ps.get(1).text()); |
||||
book.setNewestChapterTitle(""); |
||||
book.setSource(BookSource.bijian.toString()); |
||||
SearchBookBean sbb = new SearchBookBean(book.getName(), book.getAuthor()); |
||||
books.add(sbb, book); |
||||
} |
||||
return books; |
||||
} |
||||
|
||||
} |
||||
@ -0,0 +1,135 @@ |
||||
package xyz.fycz.myreader.webapi.crawler.read; |
||||
|
||||
import android.text.Html; |
||||
|
||||
import org.jsoup.Jsoup; |
||||
import org.jsoup.nodes.Document; |
||||
import org.jsoup.nodes.Element; |
||||
import org.jsoup.select.Elements; |
||||
|
||||
import java.util.ArrayList; |
||||
|
||||
import xyz.fycz.myreader.entity.SearchBookBean; |
||||
import xyz.fycz.myreader.enums.BookSource; |
||||
import xyz.fycz.myreader.greendao.entity.Book; |
||||
import xyz.fycz.myreader.greendao.entity.Chapter; |
||||
import xyz.fycz.myreader.model.mulvalmap.ConcurrentMultiValueMap; |
||||
import xyz.fycz.myreader.webapi.crawler.base.ReadCrawler; |
||||
|
||||
|
||||
public class WoLongReadCrawler implements ReadCrawler { |
||||
public static final String NAME_SPACE = "http://www.paper027.com"; |
||||
public static final String NOVEL_SEARCH = "http://www.paper027.com/search.html?keyword={key}"; |
||||
public static final String CHARSET = "UTF-8"; |
||||
public static final String SEARCH_CHARSET = "UTF-8"; |
||||
|
||||
@Override |
||||
public String getSearchLink() { |
||||
return NOVEL_SEARCH; |
||||
} |
||||
|
||||
@Override |
||||
public String getCharset() { |
||||
return CHARSET; |
||||
} |
||||
|
||||
@Override |
||||
public String getNameSpace() { |
||||
return NAME_SPACE; |
||||
} |
||||
|
||||
@Override |
||||
public Boolean isPost() { |
||||
return false; |
||||
} |
||||
|
||||
@Override |
||||
public String getSearchCharset() { |
||||
return SEARCH_CHARSET; |
||||
} |
||||
|
||||
/** |
||||
* 从html中获取章节正文 |
||||
* |
||||
* @param html |
||||
* @return |
||||
*/ |
||||
public String getContentFormHtml(String html) { |
||||
Document doc = Jsoup.parse(html); |
||||
Element divContent = doc.getElementById("contentsource"); |
||||
if (divContent != null) { |
||||
String content = Html.fromHtml(divContent.html()).toString(); |
||||
char c = 160; |
||||
String spaec = "" + c; |
||||
content = content.replace(spaec, " "); |
||||
return content; |
||||
} else { |
||||
return ""; |
||||
} |
||||
} |
||||
|
||||
/** |
||||
* 从html中获取章节列表 |
||||
* |
||||
* @param html |
||||
* @return |
||||
*/ |
||||
public ArrayList<Chapter> getChaptersFromHtml(String html) { |
||||
ArrayList<Chapter> chapters = new ArrayList<>(); |
||||
Document doc = Jsoup.parse(html); |
||||
Element divList = doc.getElementsByClass("chapters").get(0); |
||||
Elements elementsByTag = divList.getElementsByTag("a"); |
||||
int j = 0; |
||||
for (int i = elementsByTag.size() - 1; i >= 0; i--) { |
||||
Element a = elementsByTag.get(i); |
||||
String title = a.text(); |
||||
String url = a.attr("href"); |
||||
Chapter chapter = new Chapter(); |
||||
chapter.setNumber(j++); |
||||
chapter.setTitle(title); |
||||
chapter.setUrl(url); |
||||
chapters.add(chapter); |
||||
} |
||||
return chapters; |
||||
} |
||||
|
||||
/** |
||||
* 从搜索html中得到书列表 |
||||
<div> |
||||
<a href="http://www.paper027.com/novel/75432.html" target="_blank"><img class="img-rounded" src="http://www.paper027.com/uploads/novel/20190802/9883298e2e72ecfa53fabf0ef2e03e21.jpg"/></a> |
||||
<h2><a href="http://www.paper027.com/novel/75432.html" target="_blank">大主宰之混子日常</a></h2> |
||||
<p class="text-muted"><span>錯過过错</span> <small>2019-08-06 19:14</small></p> |
||||
</div> |
||||
<div class="clearfix searchresult-info"> |
||||
<p><a href="http://www.paper027.com/novel/75432.html" target="_blank">作者很懒，什么也没有留下。...</a></p> |
||||
<ul class="list-inline text-muted"> |
||||
<li>11635人看过</li> |
||||
<li>标签：</li> |
||||
<li><a class="text-warning">同人衍生</a></li> |
||||
</ul> |
||||
</div> |
||||
*/ |
||||
public ConcurrentMultiValueMap<SearchBookBean, Book> getBooksFromSearchHtml(String html) { |
||||
ConcurrentMultiValueMap<SearchBookBean, Book> books = new ConcurrentMultiValueMap<>(); |
||||
Document doc = Jsoup.parse(html); |
||||
Elements elementsByTag = doc.getElementsByClass("searchresult"); |
||||
for (int i = 0; i < elementsByTag.size(); i++) { |
||||
Element element = elementsByTag.get(i); |
||||
Elements as = element.getElementsByTag("a"); |
||||
Elements ps = element.getElementsByTag("p"); |
||||
Book book = new Book(); |
||||
book.setImgUrl(element.getElementsByTag("img").attr("src")); |
||||
book.setName(as.get(1).text()); |
||||
book.setAuthor(ps.get(0).getElementsByTag("span").get(0).text()); |
||||
book.setType(as.get(3).text()); |
||||
book.setChapterUrl(as.get(1).attr("href").replace("novel", "home/chapter/lists/id")); |
||||
book.setDesc(as.get(2).text()); |
||||
book.setNewestChapterTitle(""); |
||||
book.setSource(BookSource.wolong.toString()); |
||||
SearchBookBean sbb = new SearchBookBean(book.getName(), book.getAuthor()); |
||||
books.add(sbb, book); |
||||
} |
||||
return books; |
||||
} |
||||
|
||||
} |
||||
@ -0,0 +1,165 @@ |
||||
package xyz.fycz.myreader.webapi.crawler.read; |
||||
|
||||
import android.text.Html; |
||||
|
||||
import org.jsoup.Jsoup; |
||||
import org.jsoup.nodes.Document; |
||||
import org.jsoup.nodes.Element; |
||||
import org.jsoup.select.Elements; |
||||
|
||||
import java.util.ArrayList; |
||||
|
||||
import xyz.fycz.myreader.entity.SearchBookBean; |
||||
import xyz.fycz.myreader.enums.BookSource; |
||||
import xyz.fycz.myreader.greendao.entity.Book; |
||||
import xyz.fycz.myreader.greendao.entity.Chapter; |
||||
import xyz.fycz.myreader.model.mulvalmap.ConcurrentMultiValueMap; |
||||
import xyz.fycz.myreader.webapi.crawler.base.ReadCrawler; |
||||
|
||||
|
||||
public class YanQingLouReadCrawler implements ReadCrawler { |
||||
public static final String NAME_SPACE = "http://www.yanqinglou.com"; |
||||
public static final String NOVEL_SEARCH = "http://www.yanqinglou.com/Home/Search,action=search&q={key}"; |
||||
public static final String CHARSET = "UTF-8"; |
||||
public static final String SEARCH_CHARSET = "UTF-8"; |
||||
|
||||
@Override |
||||
public String getSearchLink() { |
||||
return NOVEL_SEARCH; |
||||
} |
||||
|
||||
@Override |
||||
public String getCharset() { |
||||
return CHARSET; |
||||
} |
||||
|
||||
@Override |
||||
public String getNameSpace() { |
||||
return NAME_SPACE; |
||||
} |
||||
|
||||
@Override |
||||
public Boolean isPost() { |
||||
return true; |
||||
} |
||||
|
||||
@Override |
||||
public String getSearchCharset() { |
||||
return SEARCH_CHARSET; |
||||
} |
||||
|
||||
/** |
||||
* 从html中获取章节正文 |
||||
* |
||||
* @param html |
||||
* @return |
||||
*/ |
||||
public String getContentFormHtml(String html) { |
||||
Document doc = Jsoup.parse(html); |
||||
Element divContent = doc.getElementById("content"); |
||||
if (divContent != null) { |
||||
String content = Html.fromHtml(divContent.html()).toString(); |
||||
char c = 160; |
||||
String spaec = "" + c; |
||||
content = content.replace(spaec, " "); |
||||
return content; |
||||
} else { |
||||
return ""; |
||||
} |
||||
} |
||||
|
||||
/** |
||||
* 从html中获取章节列表 |
||||
* |
||||
* @param html |
||||
* @return |
||||
*/ |
||||
public ArrayList<Chapter> getChaptersFromHtml(String html) { |
||||
ArrayList<Chapter> chapters = new ArrayList<>(); |
||||
Document doc = Jsoup.parse(html); |
||||
Element divList = doc.getElementsByClass("fulllistall").first(); |
||||
Elements elementsByTag = divList.getElementsByTag("a"); |
||||
for (int i = 0; i < elementsByTag.size(); i++) { |
||||
Element a = elementsByTag.get(i); |
||||
String title = a.text(); |
||||
String url = a.attr("href"); |
||||
Chapter chapter = new Chapter(); |
||||
chapter.setNumber(i); |
||||
chapter.setTitle(title); |
||||
chapter.setUrl(NAME_SPACE + url); |
||||
chapters.add(chapter); |
||||
} |
||||
return chapters; |
||||
} |
||||
|
||||
/** |
||||
* 从搜索html中得到书列表 |
||||
<div class="bookbox"> |
||||
<div class="p10"><span class="num"> <a title="斗罗之大主宰" href="/hunlian/459337/"><img layout="fixed" width="90" height="120" src="https://www.biquduo.com/files/article/image/64/64075/64075s.jpg" alt="斗罗之大主宰" /></a> </span> |
||||
<div class="bookinfo"> |
||||
<h4 class="bookname"><a title="斗罗之大主宰" href="/hunlian/459337/">斗罗之大主宰</a></h4> |
||||
<div class="author">作者：<a href="/writter/%E4%B8%8A%E5%BC%A6666.html" target="_blank" title="上弦666的作品大全">上弦666</a></div> |
||||
<div class="author">分类：婚恋</div> |
||||
<div class="cat"><span>更新到：</span><a href="/hunlian/459337/">最近更新>></a></div> |
||||
<div class="update"><span>简介：</span>叶辰，武魂，昊天锤的变异，大须弥锤！昊天锤本就是大陆第一器武魂，而武魂是大须弥锤的叶辰，会有多强？当</div> |
||||
</div> |
||||
<div class="delbutton"> <a class="del_but" title="斗罗之大主宰" href="/hunlian/459337/">阅读</a></div> |
||||
</div> |
||||
</div> |
||||
*/ |
||||
public ConcurrentMultiValueMap<SearchBookBean, Book> getBooksFromSearchHtml(String html) { |
||||
ConcurrentMultiValueMap<SearchBookBean, Book> books = new ConcurrentMultiValueMap<>(); |
||||
Document doc = Jsoup.parse(html); |
||||
String bookName = doc.select("meta[property=og:novel:book_name]").attr("content"); |
||||
if ("".equals(bookName)) { |
||||
Element div = doc.getElementsByClass("keywords").first(); |
||||
Elements divs = div.getElementsByClass("bookbox"); |
||||
for (Element divBook : divs) { |
||||
Elements as = divBook.getElementsByTag("a"); |
||||
Book book = new Book(); |
||||
book.setName(as.get(1).text()); |
||||
book.setAuthor(as.get(2).text()); |
||||
book.setType(divBook.getElementsByClass("author").get(1).text().replace("分类：", "")); |
||||
book.setNewestChapterTitle(as.get(3).text().replace("最近更新>>", "")); |
||||
book.setDesc(divBook.getElementsByClass("update").first().text().replace("简介：", "")); |
||||
book.setImgUrl(divBook.getElementsByTag("img").attr("src")); |
||||
book.setChapterUrl(NAME_SPACE + as.get(0).attr("href")); |
||||
book.setSource(BookSource.yanqinglou.toString()); |
||||
SearchBookBean sbb = new SearchBookBean(book.getName(), book.getAuthor()); |
||||
books.add(sbb, book); |
||||
} |
||||
}else { |
||||
Book book = new Book(); |
||||
getBookInfo(doc, book); |
||||
SearchBookBean sbb = new SearchBookBean(book.getName(), book.getAuthor()); |
||||
books.add(sbb, book); |
||||
} |
||||
return books; |
||||
} |
||||
|
||||
private void getBookInfo(Document doc, Book book) { |
||||
String name = doc.select("meta[property=og:novel:book_name]").attr("content"); |
||||
book.setName(name); |
||||
|
||||
String author = doc.select("meta[property=og:novel:author]").attr("content"); |
||||
book.setAuthor(author); |
||||
|
||||
String type = doc.select("meta[property=og:novel:category]").attr("content"); |
||||
book.setType(type); |
||||
|
||||
String newestTitle = doc.select("meta[property=og:novel:latest_chapter_name]").attr("content"); |
||||
book.setNewestChapterTitle(newestTitle); |
||||
|
||||
String desc = doc.select("meta[property=og:description]").attr("content"); |
||||
book.setDesc(desc); |
||||
|
||||
String img = doc.select("meta[property=og:image]").attr("content"); |
||||
book.setImgUrl(NAME_SPACE + img); |
||||
|
||||
String url = doc.select("meta[property=og:novel:read_url]").attr("content"); |
||||
book.setChapterUrl(url); |
||||
book.setSource(BookSource.yanqinglou.toString()); |
||||
} |
||||
|
||||
|
||||
} |
||||
@ -1,2 +1,2 @@ |
||||
#Mon Jan 25 21:29:58 CST 2021 |
||||
VERSION_CODE=180 |
||||
#Mon Feb 01 19:33:27 CST 2021 |
||||
VERSION_CODE=181 |
||||
|
||||
Loading…
Reference in new issue