You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 
 
FYReader/app/src/main/java/xyz/fycz/myreader/crawler/BiQuGeReadCrawler.java

215 lines
7.1 KiB

package xyz.fycz.myreader.crawler;
import android.text.Html;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import xyz.fycz.myreader.callback.ResultCallback;
import xyz.fycz.myreader.entity.SearchBookBean;
import xyz.fycz.myreader.entity.bookstore.BookType;
import xyz.fycz.myreader.enums.BookSource;
import xyz.fycz.myreader.greendao.entity.Book;
import xyz.fycz.myreader.greendao.entity.Chapter;
import xyz.fycz.myreader.mulvalmap.ConcurrentMultiValueMap;
import xyz.fycz.myreader.util.StringHelper;
import xyz.fycz.myreader.webapi.CommonApi;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class BiQuGeReadCrawler implements ReadCrawler, BookInfoCrawler {
private static final String NAME_SPACE = "https://www.52bqg.com";
private static final String NOVEL_SEARCH = "https://www.52bqg.com/modules/article/search.php";
private static final String SEARCH_KEY = "searchkey";
private static final String CHARSET = "GBK";
private ReadCrawler rc = new TianLaiReadCrawler();
@Override
public String getSearchLink() {
return NOVEL_SEARCH;
}
@Override
public String getNameSpace() {
return NAME_SPACE;
}
@Override
public String getSearchKey() {
return SEARCH_KEY;
}
@Override
public String getCharset() {
return CHARSET;
}
/**
* 获取书城小说分类列表
*
* @param html
* @return
*/
public static List<BookType> getBookTypeList(String html) {
List<BookType> bookTypes = new ArrayList<>();
Document doc = Jsoup.parse(html);
Elements divs = doc.getElementsByClass("nav");
if (divs.size() > 0) {
Elements uls = divs.get(0).getElementsByTag("ul");
if (uls.size() > 0) {
for (Element li : uls.get(0).children()) {
Element a = li.child(0);
BookType bookType = new BookType();
bookType.setTypeName(a.attr("title"));
bookType.setUrl(a.attr("href"));
if (!bookType.getTypeName().contains("小说") || bookType.getTypeName().contains("排行")) continue;
if (!StringHelper.isEmpty(bookType.getTypeName())) {
bookTypes.add(bookType);
}
}
}
}
return bookTypes;
}
/**
* 获取某一分类小说排行榜列表
*
* @param html
* @return
*/
public static List<Book> getBookRankList(String html) {
List<Book> books = new ArrayList<>();
Document doc = Jsoup.parse(html);
Elements divs = doc.getElementsByClass("r");
if (divs.size() > 0) {
Elements uls = divs.get(0).getElementsByTag("ul");
if (uls.size() > 0) {
for (Element li : uls.get(0).children()) {
Book book = new Book();
Element scanS1 = li.getElementsByClass("s1").get(0);
Element scanS2 = li.getElementsByClass("s2").get(0);
Element scanS5 = li.getElementsByClass("s5").get(0);
book.setType(scanS1.html().replace("[", "").replace("]", ""));
Element a = scanS2.getElementsByTag("a").get(0);
book.setName(a.attr("title"));
book.setChapterUrl(a.attr("href"));
book.setAuthor(scanS5.html());
book.setSource(BookSource.biquge.toString());
books.add(book);
}
}
}
return books;
}
@Override
public String getContentFormHtml(String html) {
return rc.getContentFormHtml(html);
}
@Override
public ArrayList<Chapter> getChaptersFromHtml(String html) {
return rc.getChaptersFromHtml(html);
}
/**
* 从搜索html中得到书列表
*
* @param html
* @return
*/
public ConcurrentMultiValueMap<SearchBookBean, Book> getBooksFromSearchHtml(String html) {
final ConcurrentMultiValueMap<SearchBookBean, Book> books = new ConcurrentMultiValueMap<>();
Document doc = Jsoup.parse(html);
String urlType = doc.select("meta[property=og:type]").attr("content");
if ("novel".equals(urlType)) {
String readUrl = doc.select("meta[property=og:novel:read_url]").attr("content");
Book book = new Book();
book.setChapterUrl(readUrl);
getBookInfo(html, book);
SearchBookBean sbb = new SearchBookBean(book.getName(), book.getAuthor());
books.add(sbb, book);
} else {
Elements divs = doc.getElementsByClass("novelslistss");
Element div = divs.get(0);
Elements elementsByTag = div.getElementsByTag("li");
for (Element element : elementsByTag) {
Book book = new Book();
Elements info = element.getElementsByTag("span");
book.setName(info.get(1).text());
book.setChapterUrl(info.get(1).getElementsByTag("a").attr("href"));
book.setAuthor(info.get(3).text());
book.setNewestChapterTitle(info.get(2).text());
book.setSource(BookSource.biquge.toString());
book.setType(info.get(0).text());
SearchBookBean sbb = new SearchBookBean(book.getName(), book.getAuthor());
books.add(sbb, book);
}
}
return books;
}
/**
* 获取小说详细信息
*
* @param html
* @return
*/
public Book getBookInfo(String html, Book book) {
//小说源
book.setSource(BookSource.biquge.toString());
Document doc = Jsoup.parse(html);
//图片url
Element divImg = doc.getElementById("fmimg");
Element img = divImg.getElementsByTag("img").get(0);
book.setImgUrl(img.attr("src"));
Element divInfo = doc.getElementById("info");
//书名
Element h1 = divInfo.getElementsByTag("h1").get(0);
book.setName(h1.html());
Elements ps = divInfo.getElementsByTag("p");
//作者
Element p0 = ps.get(0);
Element a = p0.getElementsByTag("a").get(0);
book.setAuthor(a.html());
//更新时间
Element p2 = ps.get(2);
Pattern pattern = Pattern.compile("更新时间:(.*)&nbsp;");
Matcher matcher = pattern.matcher(p2.html());
if (matcher.find()) {
book.setUpdateDate(matcher.group(1));
}
//最新章节
Element p3 = ps.get(3);
a = p3.getElementsByTag("a").get(0);
book.setNewestChapterTitle(a.attr("title"));
book.setNewestChapterUrl(book.getChapterUrl() + a.attr("href"));
//类型
String type = doc.select("meta[property=og:novel:category]").attr("content");
book.setType(type);
//简介
Element divIntro = doc.getElementById("intro");
book.setDesc(Html.fromHtml(divIntro.html()).toString());
return book;
}
}