优化书源结构

app/src/main/java/xyz/fycz/myreader/common/APPCONST.java

@@ -121,4 +121,6 @@ public class APPCONST {
 
     public static final ScriptEngine SCRIPT_ENGINE = new ScriptEngineManager().getEngineByName("rhino");
 
+    public static final String DEFAULT_USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4168.3 Safari/537.36";
+
 }

app/src/main/java/xyz/fycz/myreader/ui/activity/SourceLoginActivity.java

@@ -60,6 +60,7 @@ public class SourceLoginActivity extends BaseActivity {
         settings.setSupportZoom(true);
         settings.setBuiltInZoomControls(true);
         settings.setDefaultTextEncodingName("UTF-8");
+        //settings.setUserAgentString(APPCONST.DEFAULT_USER_AGENT);
         settings.setJavaScriptEnabled(true);
         CookieManager cookieManager = CookieManager.getInstance();
         binding.webView.setWebViewClient(new WebViewClient() {

app/src/main/java/xyz/fycz/myreader/util/utils/OkHttpUtils.java

@@ -18,6 +18,7 @@ import org.jsoup.nodes.Document;
 import io.reactivex.Observable;
 import okhttp3.*;
 import xyz.fycz.myreader.application.App;
+import xyz.fycz.myreader.common.APPCONST;
 import xyz.fycz.myreader.entity.StrResponse;
 import xyz.fycz.myreader.greendao.DbManager;
 import xyz.fycz.myreader.greendao.entity.CookieBean;
@@ -107,10 +108,14 @@ public class OkHttpUtils {
                 .addHeader("Connection", "Keep-Alive")
                 //.addHeader("Charsert", "utf-8")
                 .addHeader("Cache-Control", "no-cache")
-                .addHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.85 Safari/537.36");
+                .addHeader("User-Agent", APPCONST.DEFAULT_USER_AGENT);
         if (headers != null) {
             for (String name : headers.keySet()) {
-                builder.addHeader(name, headers.get(name));
+                if (name.toLowerCase().equals("user-agent")) {
+                    builder.header(name, headers.get(name));
+                } else {
+                    builder.addHeader(name, headers.get(name));
+                }
             }
         }
         if (requestBody != null) {
@@ -129,7 +134,7 @@ public class OkHttpUtils {
 
     public static InputStream getInputStream(String url) throws IOException {
         Request.Builder builder = new Request.Builder()
-                .addHeader("user-agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4168.3 Safari/537.36");
+                .addHeader("user-agent", APPCONST.DEFAULT_USER_AGENT);
         Request request = builder
                 .url(url)
                 .build();
@@ -150,7 +155,7 @@ public class OkHttpUtils {
         String url = "https://www.yuque.com/api/docs/" + key + "?book_id=19981967&include_contributors=true&include_hits=true&include_like=true&include_pager=true&include_suggests=true";
         String referer = "https://www.yuque.com/books/share/bf61f5fb-6eff-4740-ab38-749300e79306/" + key;
         Request.Builder builder = new Request.Builder()
-                .addHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4168.3 Safari/537.36")
+                .addHeader("User-Agent", APPCONST.DEFAULT_USER_AGENT)
                 .addHeader("Content-Type", "application/json")
                 .addHeader("Referer", referer);
         Request request = builder

app/src/main/java/xyz/fycz/myreader/webapi/ThirdSourceApi.java

@@ -42,9 +42,11 @@ public class ThirdSourceApi {
      */
     public static Observable<ConMVMap<SearchBookBean, Book>> searchByTC(String key, final ThirdCrawler rc) {
         try {
+            Map<String, String> headers = rc.getHeaders();
+            headers.putAll(getCookies(rc.getNameSpace()));
             BookSource source = rc.getSource();
             AnalyzeUrl analyzeUrl = new AnalyzeUrl(source.getSearchRule().getSearchUrl(),
-                    key, 1, getCookies(source), source.getSourceUrl());
+                    key, 1, headers, source.getSourceUrl());
             BookList bookList = new BookList(source.getSourceUrl(), source.getSourceName(), source, false);
             return OkHttpUtils.getStrResponse(analyzeUrl).flatMap(bookList::analyzeSearchBook)
                     .flatMap((Function<List<Book>, ObservableSource<ConMVMap<SearchBookBean, Book>>>) books -> Observable.create((ObservableOnSubscribe<ConMVMap<SearchBookBean, Book>>) emitter -> {
@@ -63,7 +65,9 @@ public class ThirdSourceApi {
             return bookInfo.analyzeBookInfo(book.getBookInfoBean().getBookInfoHtml(), bookShelfBean);
         }*/
         try {
-            AnalyzeUrl analyzeUrl = new AnalyzeUrl(book.getInfoUrl(), getCookies(source), source.getSourceUrl());
+            Map<String, String> headers = rc.getHeaders();
+            headers.putAll(getCookies(rc.getNameSpace()));
+            AnalyzeUrl analyzeUrl = new AnalyzeUrl(book.getInfoUrl(), headers, source.getSourceUrl());
             return OkHttpUtils.getStrResponse(analyzeUrl)
                     .flatMap(response -> OkHttpUtils.setCookie(response, source.getSourceUrl()))
                     .flatMap(response -> bookInfo.analyzeBookInfo(response.body(), book));
@@ -79,10 +83,12 @@ public class ThirdSourceApi {
             return bookChapterList.analyzeChapterList(bookShelfBean.getBookInfoBean().getChapterListHtml(), bookShelfBean, headerMap);
         }*/
         try {
-            AnalyzeUrl analyzeUrl = new AnalyzeUrl(book.getChapterUrl(), getCookies(source), book.getInfoUrl());
+            Map<String, String> headers = rc.getHeaders();
+            headers.putAll(getCookies(rc.getNameSpace()));
+            AnalyzeUrl analyzeUrl = new AnalyzeUrl(book.getChapterUrl(), headers, book.getInfoUrl());
             return OkHttpUtils.getStrResponse(analyzeUrl)
                     .flatMap(response -> OkHttpUtils.setCookie(response, source.getSourceUrl()))
-                    .flatMap(response -> bookChapterList.analyzeChapterList(response.body(), book, getCookies(source)))
+                    .flatMap(response -> bookChapterList.analyzeChapterList(response.body(), book, headers))
                     .flatMap(chapters -> Observable.create(emitter -> {
                         for (int i = 0; i < chapters.size(); i++) {
                             Chapter chapter = chapters.get(i);
@@ -104,7 +110,9 @@ public class ThirdSourceApi {
             return bookContent.analyzeBookContent(bookShelfBean.getBookInfoBean().getChapterListHtml(), chapterBean, nextChapterBean, bookShelfBean, headerMap);
         }*/
         try {
-            AnalyzeUrl analyzeUrl = new AnalyzeUrl(chapter.getUrl(), getCookies(source), book.getChapterUrl());
+            Map<String, String> headers = rc.getHeaders();
+            headers.putAll(getCookies(rc.getNameSpace()));
+            AnalyzeUrl analyzeUrl = new AnalyzeUrl(chapter.getUrl(), headers, book.getChapterUrl());
             String contentRule = source.getContentRule().getContent();
             if (contentRule.startsWith("$") && !contentRule.startsWith("$.")) {
                 //动态网页第一个js放到webView里执行
@@ -120,11 +128,11 @@ public class ThirdSourceApi {
                     }
                 }
                 return OkHttpUtils.getAjaxString(analyzeUrl, source.getSourceUrl(), js)
-                        .flatMap(response -> bookContent.analyzeBookContent(response, chapter, null, book, getCookies(source)));
+                        .flatMap(response -> bookContent.analyzeBookContent(response, chapter, null, book, headers));
             } else {
                 return OkHttpUtils.getStrResponse(analyzeUrl)
                         .flatMap(response -> OkHttpUtils.setCookie(response, source.getSourceUrl()))
-                        .flatMap(response -> bookContent.analyzeBookContent(response, chapter, null, book, getCookies(source)));
+                        .flatMap(response -> bookContent.analyzeBookContent(response, chapter, null, book, headers));
             }
         } catch (Exception e) {
             return Observable.error(new Throwable(String.format("url错误:%s", chapter.getUrl())));

app/src/main/java/xyz/fycz/myreader/webapi/crawler/base/BaseReadCrawler.java

@@ -6,6 +6,7 @@ import java.util.List;
 import java.util.Map;
 
 import io.reactivex.Observable;
+import xyz.fycz.myreader.common.APPCONST;
 import xyz.fycz.myreader.entity.SearchBookBean;
 import xyz.fycz.myreader.entity.StrResponse;
 import xyz.fycz.myreader.greendao.entity.Book;
@@ -14,13 +15,16 @@ import xyz.fycz.myreader.model.mulvalmap.ConMVMap;
 
 /**
  * 因新版书源使用StrResponse，为了兼容旧版本，书源全部继承自此类
+ *
  * @author fengyue
  * @date 2021/5/13 22:29
  */
 public abstract class BaseReadCrawler implements ReadCrawler {
     @Override
     public Map<String, String> getHeaders() {
-        return new HashMap<>();
+        Map<String, String> headers = new HashMap<>();
+        headers.put("User-Agent", APPCONST.DEFAULT_USER_AGENT);
+        return headers;
     }
 
     @Override
@@ -41,8 +45,8 @@ public abstract class BaseReadCrawler implements ReadCrawler {
     @Override
     public Observable<ConMVMap<SearchBookBean, Book>> getBooksFromStrResponse(StrResponse response) {
         return Observable.create(emitter -> {
-           emitter.onNext(getBooksFromSearchHtml(response.body()));
+            emitter.onNext(getBooksFromSearchHtml(response.body()));
-           emitter.onComplete();
+            emitter.onComplete();
         });
     }
 

app/src/main/java/xyz/fycz/myreader/webapi/crawler/source/ThirdCrawler.java

@@ -56,11 +56,6 @@ public class ThirdCrawler extends BaseReadCrawler implements BookInfoCrawler {
         return null;
     }
 
-    @Override
-    public Map<String, String> getHeaders() {
-        return null;
-    }
-
     public ConMVMap<SearchBookBean, Book> getBooks(List<Book> books) {
         ConMVMap<SearchBookBean, Book> newBooks = new ConMVMap<>();
         for (Book book : books){