diff --git a/app/src/main/java/xyz/fycz/myreader/common/APPCONST.java b/app/src/main/java/xyz/fycz/myreader/common/APPCONST.java index 4b8c0b0..6e6b8c8 100644 --- a/app/src/main/java/xyz/fycz/myreader/common/APPCONST.java +++ b/app/src/main/java/xyz/fycz/myreader/common/APPCONST.java @@ -121,4 +121,6 @@ public class APPCONST { public static final ScriptEngine SCRIPT_ENGINE = new ScriptEngineManager().getEngineByName("rhino"); + public static final String DEFAULT_USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4168.3 Safari/537.36"; + } diff --git a/app/src/main/java/xyz/fycz/myreader/ui/activity/SourceLoginActivity.java b/app/src/main/java/xyz/fycz/myreader/ui/activity/SourceLoginActivity.java index 98e02f9..0313386 100644 --- a/app/src/main/java/xyz/fycz/myreader/ui/activity/SourceLoginActivity.java +++ b/app/src/main/java/xyz/fycz/myreader/ui/activity/SourceLoginActivity.java @@ -60,6 +60,7 @@ public class SourceLoginActivity extends BaseActivity { settings.setSupportZoom(true); settings.setBuiltInZoomControls(true); settings.setDefaultTextEncodingName("UTF-8"); + //settings.setUserAgentString(APPCONST.DEFAULT_USER_AGENT); settings.setJavaScriptEnabled(true); CookieManager cookieManager = CookieManager.getInstance(); binding.webView.setWebViewClient(new WebViewClient() { diff --git a/app/src/main/java/xyz/fycz/myreader/util/utils/OkHttpUtils.java b/app/src/main/java/xyz/fycz/myreader/util/utils/OkHttpUtils.java index e204587..01ada23 100644 --- a/app/src/main/java/xyz/fycz/myreader/util/utils/OkHttpUtils.java +++ b/app/src/main/java/xyz/fycz/myreader/util/utils/OkHttpUtils.java @@ -18,6 +18,7 @@ import org.jsoup.nodes.Document; import io.reactivex.Observable; import okhttp3.*; import xyz.fycz.myreader.application.App; +import xyz.fycz.myreader.common.APPCONST; import xyz.fycz.myreader.entity.StrResponse; import xyz.fycz.myreader.greendao.DbManager; import xyz.fycz.myreader.greendao.entity.CookieBean; @@ -107,10 +108,14 @@ public class OkHttpUtils { .addHeader("Connection", "Keep-Alive") //.addHeader("Charsert", "utf-8") .addHeader("Cache-Control", "no-cache") - .addHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.85 Safari/537.36"); + .addHeader("User-Agent", APPCONST.DEFAULT_USER_AGENT); if (headers != null) { for (String name : headers.keySet()) { - builder.addHeader(name, headers.get(name)); + if (name.toLowerCase().equals("user-agent")) { + builder.header(name, headers.get(name)); + } else { + builder.addHeader(name, headers.get(name)); + } } } if (requestBody != null) { @@ -129,7 +134,7 @@ public class OkHttpUtils { public static InputStream getInputStream(String url) throws IOException { Request.Builder builder = new Request.Builder() - .addHeader("user-agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4168.3 Safari/537.36"); + .addHeader("user-agent", APPCONST.DEFAULT_USER_AGENT); Request request = builder .url(url) .build(); @@ -150,7 +155,7 @@ public class OkHttpUtils { String url = "https://www.yuque.com/api/docs/" + key + "?book_id=19981967&include_contributors=true&include_hits=true&include_like=true&include_pager=true&include_suggests=true"; String referer = "https://www.yuque.com/books/share/bf61f5fb-6eff-4740-ab38-749300e79306/" + key; Request.Builder builder = new Request.Builder() - .addHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4168.3 Safari/537.36") + .addHeader("User-Agent", APPCONST.DEFAULT_USER_AGENT) .addHeader("Content-Type", "application/json") .addHeader("Referer", referer); Request request = builder diff --git a/app/src/main/java/xyz/fycz/myreader/webapi/ThirdSourceApi.java b/app/src/main/java/xyz/fycz/myreader/webapi/ThirdSourceApi.java index 3e9141e..5118839 100644 --- a/app/src/main/java/xyz/fycz/myreader/webapi/ThirdSourceApi.java +++ b/app/src/main/java/xyz/fycz/myreader/webapi/ThirdSourceApi.java @@ -42,9 +42,11 @@ public class ThirdSourceApi { */ public static Observable> searchByTC(String key, final ThirdCrawler rc) { try { + Map headers = rc.getHeaders(); + headers.putAll(getCookies(rc.getNameSpace())); BookSource source = rc.getSource(); AnalyzeUrl analyzeUrl = new AnalyzeUrl(source.getSearchRule().getSearchUrl(), - key, 1, getCookies(source), source.getSourceUrl()); + key, 1, headers, source.getSourceUrl()); BookList bookList = new BookList(source.getSourceUrl(), source.getSourceName(), source, false); return OkHttpUtils.getStrResponse(analyzeUrl).flatMap(bookList::analyzeSearchBook) .flatMap((Function, ObservableSource>>) books -> Observable.create((ObservableOnSubscribe>) emitter -> { @@ -63,7 +65,9 @@ public class ThirdSourceApi { return bookInfo.analyzeBookInfo(book.getBookInfoBean().getBookInfoHtml(), bookShelfBean); }*/ try { - AnalyzeUrl analyzeUrl = new AnalyzeUrl(book.getInfoUrl(), getCookies(source), source.getSourceUrl()); + Map headers = rc.getHeaders(); + headers.putAll(getCookies(rc.getNameSpace())); + AnalyzeUrl analyzeUrl = new AnalyzeUrl(book.getInfoUrl(), headers, source.getSourceUrl()); return OkHttpUtils.getStrResponse(analyzeUrl) .flatMap(response -> OkHttpUtils.setCookie(response, source.getSourceUrl())) .flatMap(response -> bookInfo.analyzeBookInfo(response.body(), book)); @@ -79,10 +83,12 @@ public class ThirdSourceApi { return bookChapterList.analyzeChapterList(bookShelfBean.getBookInfoBean().getChapterListHtml(), bookShelfBean, headerMap); }*/ try { - AnalyzeUrl analyzeUrl = new AnalyzeUrl(book.getChapterUrl(), getCookies(source), book.getInfoUrl()); + Map headers = rc.getHeaders(); + headers.putAll(getCookies(rc.getNameSpace())); + AnalyzeUrl analyzeUrl = new AnalyzeUrl(book.getChapterUrl(), headers, book.getInfoUrl()); return OkHttpUtils.getStrResponse(analyzeUrl) .flatMap(response -> OkHttpUtils.setCookie(response, source.getSourceUrl())) - .flatMap(response -> bookChapterList.analyzeChapterList(response.body(), book, getCookies(source))) + .flatMap(response -> bookChapterList.analyzeChapterList(response.body(), book, headers)) .flatMap(chapters -> Observable.create(emitter -> { for (int i = 0; i < chapters.size(); i++) { Chapter chapter = chapters.get(i); @@ -104,7 +110,9 @@ public class ThirdSourceApi { return bookContent.analyzeBookContent(bookShelfBean.getBookInfoBean().getChapterListHtml(), chapterBean, nextChapterBean, bookShelfBean, headerMap); }*/ try { - AnalyzeUrl analyzeUrl = new AnalyzeUrl(chapter.getUrl(), getCookies(source), book.getChapterUrl()); + Map headers = rc.getHeaders(); + headers.putAll(getCookies(rc.getNameSpace())); + AnalyzeUrl analyzeUrl = new AnalyzeUrl(chapter.getUrl(), headers, book.getChapterUrl()); String contentRule = source.getContentRule().getContent(); if (contentRule.startsWith("$") && !contentRule.startsWith("$.")) { //动态网页第一个js放到webView里执行 @@ -120,11 +128,11 @@ public class ThirdSourceApi { } } return OkHttpUtils.getAjaxString(analyzeUrl, source.getSourceUrl(), js) - .flatMap(response -> bookContent.analyzeBookContent(response, chapter, null, book, getCookies(source))); + .flatMap(response -> bookContent.analyzeBookContent(response, chapter, null, book, headers)); } else { return OkHttpUtils.getStrResponse(analyzeUrl) .flatMap(response -> OkHttpUtils.setCookie(response, source.getSourceUrl())) - .flatMap(response -> bookContent.analyzeBookContent(response, chapter, null, book, getCookies(source))); + .flatMap(response -> bookContent.analyzeBookContent(response, chapter, null, book, headers)); } } catch (Exception e) { return Observable.error(new Throwable(String.format("url错误:%s", chapter.getUrl()))); diff --git a/app/src/main/java/xyz/fycz/myreader/webapi/crawler/base/BaseReadCrawler.java b/app/src/main/java/xyz/fycz/myreader/webapi/crawler/base/BaseReadCrawler.java index f385976..8031b54 100644 --- a/app/src/main/java/xyz/fycz/myreader/webapi/crawler/base/BaseReadCrawler.java +++ b/app/src/main/java/xyz/fycz/myreader/webapi/crawler/base/BaseReadCrawler.java @@ -6,6 +6,7 @@ import java.util.List; import java.util.Map; import io.reactivex.Observable; +import xyz.fycz.myreader.common.APPCONST; import xyz.fycz.myreader.entity.SearchBookBean; import xyz.fycz.myreader.entity.StrResponse; import xyz.fycz.myreader.greendao.entity.Book; @@ -14,13 +15,16 @@ import xyz.fycz.myreader.model.mulvalmap.ConMVMap; /** * 因新版书源使用StrResponse,为了兼容旧版本,书源全部继承自此类 + * * @author fengyue * @date 2021/5/13 22:29 */ public abstract class BaseReadCrawler implements ReadCrawler { @Override public Map getHeaders() { - return new HashMap<>(); + Map headers = new HashMap<>(); + headers.put("User-Agent", APPCONST.DEFAULT_USER_AGENT); + return headers; } @Override @@ -41,8 +45,8 @@ public abstract class BaseReadCrawler implements ReadCrawler { @Override public Observable> getBooksFromStrResponse(StrResponse response) { return Observable.create(emitter -> { - emitter.onNext(getBooksFromSearchHtml(response.body())); - emitter.onComplete(); + emitter.onNext(getBooksFromSearchHtml(response.body())); + emitter.onComplete(); }); } diff --git a/app/src/main/java/xyz/fycz/myreader/webapi/crawler/source/ThirdCrawler.java b/app/src/main/java/xyz/fycz/myreader/webapi/crawler/source/ThirdCrawler.java index c9c00ef..e579d77 100644 --- a/app/src/main/java/xyz/fycz/myreader/webapi/crawler/source/ThirdCrawler.java +++ b/app/src/main/java/xyz/fycz/myreader/webapi/crawler/source/ThirdCrawler.java @@ -56,11 +56,6 @@ public class ThirdCrawler extends BaseReadCrawler implements BookInfoCrawler { return null; } - @Override - public Map getHeaders() { - return null; - } - public ConMVMap getBooks(List books) { ConMVMap newBooks = new ConMVMap<>(); for (Book book : books){