k
/
legado
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

优化

Browse Source
pull/540/head
gedoor 4 years ago
parent f3917d9d5e
commit b4ef8e6f32
8 changed files with 215 additions and 114 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 161
      app/src/main/java/io/legado/app/lib/icu4j/CharsetDetector.java
  2. 39
      app/src/main/java/io/legado/app/lib/icu4j/CharsetMatch.java
  3. 12
      app/src/main/java/io/legado/app/lib/icu4j/CharsetRecog_2022.java
  4. 6
      app/src/main/java/io/legado/app/lib/icu4j/CharsetRecog_UTF8.java
  5. 18
      app/src/main/java/io/legado/app/lib/icu4j/CharsetRecog_Unicode.java
  6. 32
      app/src/main/java/io/legado/app/lib/icu4j/CharsetRecog_mbcs.java
  7. 47
      app/src/main/java/io/legado/app/lib/icu4j/CharsetRecog_sbcs.java
  8. 14
      app/src/main/java/io/legado/app/lib/icu4j/CharsetRecognizer.java

161
app/src/main/java/io/legado/app/lib/icu4j/CharsetDetector.java
Unescape View File

@ -1,7 +1,8 @@
/* GENERATED SOURCE. DO NOT MODIFY. */
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/**
* ******************************************************************************
* Copyright (C) 2005-2014, International Business Machines Corporation and *
* Copyright (C) 2005-2016, International Business Machines Corporation and *
* others. All Rights Reserved. *
* ******************************************************************************
*/
@ -23,21 +24,21 @@ import java.util.List;
* The result of the detection operation is a list of possibly matching
* charsets, or, for simple use, you can just ask for a Java Reader that
* will will work over the input data.
* <p/>
* <p>
* Character set detection is at best an imprecise operation. The detection
* process will attempt to identify the charset that best matches the characteristics
* of the byte data, but the process is partly statistical in nature, and
* the results can not be guaranteed to always be correct.
* <p/>
* <p>
* For best accuracy in charset detection, the input data should be primarily
* in a single language, and a minimum of a few hundred bytes worth of plain text
* in the language are needed. The detection process will attempt to
* ignore html or xml style markup that could otherwise obscure the content.
* <p/>
* <p>
*
* @stable ICU 3.4
* @hide All android.icu classes are currently hidden
*/
@SuppressWarnings({"JavaDoc", "unused", "RedundantSuppression"})
public class CharsetDetector {
// Question: Should we have getters corresponding to the setters for input text
@ -51,7 +52,7 @@ public class CharsetDetector {
// wants the data, and doesn't care about a char set name.
/**
* Constructor
* Constructor
*
* @stable ICU 3.4
*/
@ -60,17 +61,18 @@ public class CharsetDetector {
/**
* Set the declared encoding for charset detection.
* The declared encoding of an input text is an encoding obtained
* from an http header or xml declaration or similar source that
* can be provided as additional information to the charset detector.
* A match between a declared encoding and a possible detected encoding
* will raise the quality of that detected encoding by a small delta,
* and will also appear as a "reason" for the match.
* <p/>
* The declared encoding of an input text is an encoding obtained
* from an http header or xml declaration or similar source that
* can be provided as additional information to the charset detector.
* A match between a declared encoding and a possible detected encoding
* will raise the quality of that detected encoding by a small delta,
* and will also appear as a "reason" for the match.
* <p>
* A declared encoding that is incompatible with the input data being
* analyzed will not be added to the list of possible encodings.
*
* @param encoding The declared encoding
* @param encoding The declared encoding
*
* @stable ICU 3.4
*/
public CharsetDetector setDeclaredEncoding(String encoding) {
@ -82,7 +84,9 @@ public class CharsetDetector {
* Set the input text (byte) data whose charset is to be detected.
*
* @param in the input text of unknown encoding
*
* @return This CharsetDetector
*
* @stable ICU 3.4
*/
public CharsetDetector setText(byte[] in) {
@ -96,15 +100,17 @@ public class CharsetDetector {
/**
* Set the input text (byte) data whose charset is to be detected.
* <p/>
* The input stream that supplies the character data must have markSupported()
* == true; the charset detection process will read a small amount of data,
* then return the stream to its original position via
* the InputStream.reset() operation. The exact amount that will
* be read depends on the characteristics of the data itself.
* <p>
* The input stream that supplies the character data must have markSupported()
* == true; the charset detection process will read a small amount of data,
* then return the stream to its original position via
* the InputStream.reset() operation. The exact amount that will
* be read depends on the characteristics of the data itself.
*
* @param in the input text of unknown encoding
*
* @return This CharsetDetector
*
* @stable ICU 3.4
*/
@ -133,20 +139,21 @@ public class CharsetDetector {
/**
* Return the charset that best matches the supplied input data.
* <p>
* Note though, that because the detection
*
* Note though, that because the detection
* only looks at the start of the input data,
* there is a possibility that the returned charset will fail to handle
* the full set of input data.
* p/>
* aise an exception if
* <ul>
* <li>no charset appears to match the data.</li>
* <li>no input text has been provided</li>
* </ul>
* <p>
* Raise an exception if
* <ul>
* <li>no charset appears to match the data.</li>
* <li>no input text has been provided</li>
* </ul>
*
* @return a CharsetMatch object representing the best matching charset, or
* <code>null</code> if there are no matches.
* <code>null</code> if there are no matches.
*
* @stable ICU 3.4
*/
public CharsetMatch detect() {
@ -154,7 +161,7 @@ public class CharsetDetector {
// detectAll(), and cut it short as soon as a match with a high confidence
// is found. This is something to be done later, after things are otherwise
// working.
CharsetMatch matches[] = detectAll();
CharsetMatch[] matches = detectAll();
if (matches == null || matches.length == 0) {
return null;
@ -164,21 +171,22 @@ public class CharsetDetector {
}
/**
* Return an array of all charsets that appear to be plausible
* matches with the input data. The array is ordered with the
* best quality match first.
* <p/>
* aise an exception if
* <ul>
* <li>no charsets appear to match the input data.</li>
* <li>no input text has been provided</li>
* </ul>
* Return an array of all charsets that appear to be plausible
* matches with the input data. The array is ordered with the
* best quality match first.
* <p>
* Raise an exception if
* <ul>
* <li>no charsets appear to match the input data.</li>
* <li>no input text has been provided</li>
* </ul>
*
* @return An array of CharsetMatch objects representing possibly matching charsets.
*
* @stable ICU 3.4
*/
public CharsetMatch[] detectAll() {
ArrayList<CharsetMatch> matches = new ArrayList<CharsetMatch>();
ArrayList<CharsetMatch> matches = new ArrayList<>();
MungeInput(); // Strip html markup, collect byte stats.
@ -205,21 +213,23 @@ public class CharsetDetector {
/**
* Autodetect the charset of an inputStream, and return a Java Reader
* to access the converted input data.
* <p/>
* <p>
* This is a convenience method that is equivalent to
* <code>this.setDeclaredEncoding(declaredEncoding).setText(in).detect().getReader();</code>
* <p/>
* For the input stream that supplies the character data, markSupported()
* must be true; the charset detection will read a small amount of data,
* then return the stream to its original position via
* the InputStream.reset() operation. The exact amount that will
* be read depends on the characteristics of the data itself.
* <p/>
* <code>this.setDeclaredEncoding(declaredEncoding).setText(in).detect().getReader();</code>
* <p>
* For the input stream that supplies the character data, markSupported()
* must be true; the charset detection will read a small amount of data,
* then return the stream to its original position via
* the InputStream.reset() operation. The exact amount that will
* be read depends on the characteristics of the data itself.
*<p>
* Raise an exception if no charsets appear to match the input data.
*
* @param in The source of the byte data in the unknown charset.
* @param declaredEncoding A declared encoding for the data, if available,
* or null or an empty string if none is available.
* @param in The source of the byte data in the unknown charset.
*
* @param declaredEncoding A declared encoding for the data, if available,
* or null or an empty string if none is available.
*
* @stable ICU 3.4
*/
public Reader getReader(InputStream in, String declaredEncoding) {
@ -243,15 +253,17 @@ public class CharsetDetector {
/**
* Autodetect the charset of an inputStream, and return a String
* containing the converted input data.
* <p/>
* <p>
* This is a convenience method that is equivalent to
* <code>this.setDeclaredEncoding(declaredEncoding).setText(in).detect().getString();</code>
* <p/>
* <code>this.setDeclaredEncoding(declaredEncoding).setText(in).detect().getString();</code>
*<p>
* Raise an exception if no charsets appear to match the input data.
*
* @param in The source of the byte data in the unknown charset.
* @param declaredEncoding A declared encoding for the data, if available,
* or null or an empty string if none is available.
* @param in The source of the byte data in the unknown charset.
*
* @param declaredEncoding A declared encoding for the data, if available,
* or null or an empty string if none is available.
*
* @stable ICU 3.4
*/
public String getString(byte[] in, String declaredEncoding) {
@ -285,6 +297,7 @@ public class CharsetDetector {
*
* @return an array of the names of all charsets supported by
* <code>CharsetDetector</code> class.
*
* @stable ICU 3.4
*/
public static String[] getAllDetectableCharsets() {
@ -299,8 +312,10 @@ public class CharsetDetector {
* Test whether or not input filtering is enabled.
*
* @return <code>true</code> if input text will be filtered.
* @stable ICU 3.4
*
* @see #enableInputFilter
*
* @stable ICU 3.4
*/
public boolean inputFilterEnabled() {
return fStripTags;
@ -308,11 +323,13 @@ public class CharsetDetector {
/**
* Enable filtering of input text. If filtering is enabled,
* text within angle brackets ("<" and ">") will be removed
* text within angle brackets ("&lt;" and "&gt;") will be removed
* before detection.
*
* @param filter <code>true</code> to enable input text filtering.
*
* @return The previous setting.
*
* @stable ICU 3.4
*/
public boolean enableInputFilter(boolean filter) {
@ -412,7 +429,7 @@ public class CharsetDetector {
int fInputLen; // Length of the byte data in fInputBytes.
short fByteStats[] = // byte frequency statistics for the input text.
short[] fByteStats = // byte frequency statistics for the input text.
new short[256]; // Value is percent, not absolute.
// Value is rounded up, so zero really means zero occurences.
@ -457,7 +474,7 @@ public class CharsetDetector {
private static final List<CSRecognizerInfo> ALL_CS_RECOGNIZERS;
static {
List<CSRecognizerInfo> list = new ArrayList<CSRecognizerInfo>();
List<CSRecognizerInfo> list = new ArrayList<>();
list.add(new CSRecognizerInfo(new CharsetRecog_UTF8(), true));
list.add(new CSRecognizerInfo(new CharsetRecog_Unicode.CharsetRecog_UTF_16_BE(), true));
@ -500,13 +517,13 @@ public class CharsetDetector {
*
* @return an array of the names of charsets that can be recognized by this CharsetDetector
* instance.
* <p>
* {@literal @}internal
*
* @internal
* @deprecated This API is ICU internal only.
*/
@Deprecated
public String[] getDetectableCharsets() {
List<String> csnames = new ArrayList<String>(ALL_CS_RECOGNIZERS.size());
List<String> csnames = new ArrayList<>(ALL_CS_RECOGNIZERS.size());
for (int i = 0; i < ALL_CS_RECOGNIZERS.size(); i++) {
CSRecognizerInfo rcinfo = ALL_CS_RECOGNIZERS.get(i);
boolean active = (fEnabledRecognizers == null) ? rcinfo.isDefaultEnabled : fEnabledRecognizers[i];
@ -514,7 +531,7 @@ public class CharsetDetector {
csnames.add(rcinfo.recognizer.getName());
}
}
return csnames.toArray(new String[csnames.size()]);
return csnames.toArray(new String[0]);
}
/**
@ -523,13 +540,13 @@ public class CharsetDetector {
* {@link #getAllDetectableCharsets()}.
*
* @param encoding the name of charset encoding.
* @param enabled <code>true</code> to enable, or <code>false</code> to disable the
* charset encoding.
* @param enabled <code>true</code> to enable, or <code>false</code> to disable the
* charset encoding.
* @return A reference to this <code>CharsetDetector</code>.
* @throws IllegalArgumentException when the name of charset encoding is
* not supported.
* <p>
* {@literal @}internal
* not supported.
*
* @internal
* @deprecated This API is ICU internal only.
*/
@Deprecated

39
app/src/main/java/io/legado/app/lib/icu4j/CharsetMatch.java
Unescape View File

@ -1,7 +1,8 @@
/* GENERATED SOURCE. DO NOT MODIFY. */
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/**
* ******************************************************************************
* Copyright (C) 2005-2012, International Business Machines Corporation and *
* Copyright (C) 2005-2016, International Business Machines Corporation and *
* others. All Rights Reserved. *
* ******************************************************************************
*/
@ -19,28 +20,29 @@ import java.io.Reader;
* as a possible encoding for a set of input data. From an instance of this
* class, you can ask for a confidence level in the charset identification,
* or for Java Reader or String to access the original byte data in Unicode form.
* <p/>
* <p>
* Instances of this class are created only by CharsetDetectors.
* <p/>
* <p>
* Note: this class has a natural ordering that is inconsistent with equals.
* The natural ordering is based on the match confidence value.
*
* @stable ICU 3.4
* @hide All android.icu classes are currently hidden
*/
@SuppressWarnings({"JavaDoc", "unused"})
public class CharsetMatch implements Comparable<CharsetMatch> {
/**
* Create a java.io.Reader for reading the Unicode character data corresponding
* to the original byte data supplied to the Charset detect operation.
* <p/>
* <p>
* CAUTION: if the source of the byte data was an InputStream, a Reader
* can be created for only one matching char set using this method. If more
* than one charset needs to be tried, the caller will need to reset
* the InputStream and create InputStreamReaders itself, based on the charset name.
*
* @return the Reader for the Unicode character data.
*
* @stable ICU 3.4
*/
public Reader getReader() {
@ -63,9 +65,10 @@ public class CharsetMatch implements Comparable<CharsetMatch> {
* to the original byte data supplied to the Charset detect operation.
*
* @return a String created from the converted input data.
*
* @stable ICU 3.4
*/
public String getString() throws IOException {
public String getString() throws java.io.IOException {
return getString(-1);
}
@ -81,9 +84,10 @@ public class CharsetMatch implements Comparable<CharsetMatch> {
* source of the data is an input stream, or -1 for
* unlimited length.
* @return a String created from the converted input data.
*
* @stable ICU 3.4
*/
public String getString(int maxLength) throws IOException {
public String getString(int maxLength) throws java.io.IOException {
String result = null;
if (fInputStream != null) {
StringBuilder sb = new StringBuilder();
@ -107,7 +111,7 @@ public class CharsetMatch implements Comparable<CharsetMatch> {
* be used to open a charset (e.g. IBM424_rtl). The ending '_rtl' or 'ltr'
* should be stripped off before creating the string.
*/
int startSuffix = name.indexOf("_rtl") < 0 ? name.indexOf("_ltr") : name.indexOf("_rtl");
int startSuffix = !name.contains("_rtl") ? name.indexOf("_ltr") : name.indexOf("_rtl");
if (startSuffix > 0) {
name = name.substring(0, startSuffix);
}
@ -124,6 +128,7 @@ public class CharsetMatch implements Comparable<CharsetMatch> {
* charset.
*
* @return the confidence in the charset match
*
* @stable ICU 3.4
*/
public int getConfidence() {
@ -138,10 +143,12 @@ public class CharsetMatch implements Comparable<CharsetMatch> {
* charsets that are registered with the IANA charset registry,
* this is the MIME-preferred registerd name.
*
* @see java.nio.charset.Charset
* @see java.io.InputStreamReader
*
* @return The name of the charset.
*
* @stable ICU 3.4
* @see java.nio.charset.Charset
* @see InputStreamReader
*/
public String getName() {
return fCharsetName;
@ -151,6 +158,7 @@ public class CharsetMatch implements Comparable<CharsetMatch> {
* Get the ISO code for the language of the detected charset.
*
* @return The ISO code for the language or <code>null</code> if the language cannot be determined.
*
* @stable ICU 3.4
*/
public String getLanguage() {
@ -160,16 +168,17 @@ public class CharsetMatch implements Comparable<CharsetMatch> {
/**
* Compare to other CharsetMatch objects.
* Comparison is based on the match confidence value, which
* allows CharsetDetector.detectAll() to order its results.
* allows CharsetDetector.detectAll() to order its results.
*
* @param other the CharsetMatch object to compare against.
* @return a negative integer, zero, or a positive integer as the
* confidence level of this CharsetMatch
* is less than, equal to, or greater than that of
* the argument.
* confidence level of this CharsetMatch
* is less than, equal to, or greater than that of
* the argument.
* @throws ClassCastException if the argument is not a CharsetMatch.
* @stable ICU 4.4
*/
@Override
public int compareTo(CharsetMatch other) {
int compareResult = 0;
if (this.fConfidence > other.fConfidence) {

12
app/src/main/java/io/legado/app/lib/icu4j/CharsetRecog_2022.java
Unescape View File

@ -1,4 +1,5 @@
/* GENERATED SOURCE. DO NOT MODIFY. */
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
*******************************************************************************
* Copyright (C) 2005 - 2012, International Business Machines Corporation and *
@ -7,7 +8,6 @@
*/
package io.legado.app.lib.icu4j;
/**
* class CharsetRecog_2022 part of the ICU charset detection imlementation.
* This is a superclass for the individual detectors for
@ -74,7 +74,7 @@ abstract class CharsetRecog_2022 extends CharsetRecognizer {
//
// Initial quality is based on relative proportion of recongized vs.
// unrecognized escape sequences.
// unrecognized escape sequences.
// All good: quality = 100;
// half or less good: quality = 0;
// linear inbetween.
@ -110,10 +110,12 @@ abstract class CharsetRecog_2022 extends CharsetRecognizer {
{0x1b, 0x2e, 0x46} // ISO 8859-7
};
@Override
String getName() {
return "ISO-2022-JP";
}
@Override
CharsetMatch match(CharsetDetector det) {
int confidence = match(det.fInputBytes, det.fInputLen, escapeSequences);
return confidence == 0 ? null : new CharsetMatch(det, this, confidence);
@ -125,10 +127,12 @@ abstract class CharsetRecog_2022 extends CharsetRecognizer {
{0x1b, 0x24, 0x29, 0x43}
};
@Override
String getName() {
return "ISO-2022-KR";
}
@Override
CharsetMatch match(CharsetDetector det) {
int confidence = match(det.fInputBytes, det.fInputLen, escapeSequences);
return confidence == 0 ? null : new CharsetMatch(det, this, confidence);
@ -150,10 +154,12 @@ abstract class CharsetRecog_2022 extends CharsetRecognizer {
{0x1b, 0x4f}, // SS3
};
@Override
String getName() {
return "ISO-2022-CN";
}
@Override
CharsetMatch match(CharsetDetector det) {
int confidence = match(det.fInputBytes, det.fInputLen, escapeSequences);
return confidence == 0 ? null : new CharsetMatch(det, this, confidence);

6
app/src/main/java/io/legado/app/lib/icu4j/CharsetRecog_UTF8.java
Unescape View File

@ -1,4 +1,5 @@
/* GENERATED SOURCE. DO NOT MODIFY. */
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/**
* ******************************************************************************
* Copyright (C) 2005 - 2014, International Business Machines Corporation and *
@ -7,12 +8,12 @@
*/
package io.legado.app.lib.icu4j;
/**
* Charset recognizer for UTF-8
*/
class CharsetRecog_UTF8 extends CharsetRecognizer {
@Override
String getName() {
return "UTF-8";
}
@ -20,6 +21,7 @@ class CharsetRecog_UTF8 extends CharsetRecognizer {
/* (non-Javadoc)
* @see com.ibm.icu.text.CharsetRecognizer#match(com.ibm.icu.text.CharsetDetector)
*/
@Override
CharsetMatch match(CharsetDetector det) {
boolean hasBOM = false;
int numValid = 0;

18
app/src/main/java/io/legado/app/lib/icu4j/CharsetRecog_Unicode.java
Unescape View File

@ -1,4 +1,5 @@
/* GENERATED SOURCE. DO NOT MODIFY. */
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
*******************************************************************************
* Copyright (C) 1996-2013, International Business Machines Corporation and *
@ -9,7 +10,6 @@
package io.legado.app.lib.icu4j;
/**
* This class matches UTF-16 and UTF-32, both big- and little-endian. The
* BOM will be used if it is present.
@ -19,11 +19,13 @@ abstract class CharsetRecog_Unicode extends CharsetRecognizer {
/* (non-Javadoc)
* @see com.ibm.icu.text.CharsetRecognizer#getName()
*/
@Override
abstract String getName();
/* (non-Javadoc)
* @see com.ibm.icu.text.CharsetRecognizer#match(com.ibm.icu.text.CharsetDetector)
*/
@Override
abstract CharsetMatch match(CharsetDetector det);
static int codeUnit16FromBytes(byte hi, byte lo) {
@ -34,7 +36,7 @@ abstract class CharsetRecog_Unicode extends CharsetRecognizer {
// Any 8 bit non-control characters bump the confidence up. These have a zero high byte,
// and are very likely to be UTF-16, although they could also be part of a UTF-32 code.
// NULs are a contra-indication, they will appear commonly if the actual encoding is UTF-32.
// NULs should be rare in actual text.
// NULs should be rare in actual text.
static int adjustConfidence(int codeUnit, int confidence) {
if (codeUnit == 0) {
confidence -= 10;
@ -50,10 +52,12 @@ abstract class CharsetRecog_Unicode extends CharsetRecognizer {
}
static class CharsetRecog_UTF_16_BE extends CharsetRecog_Unicode {
@Override
String getName() {
return "UTF-16BE";
}
@Override
CharsetMatch match(CharsetDetector det) {
byte[] input = det.fRawInput;
int confidence = 10;
@ -81,10 +85,12 @@ abstract class CharsetRecog_Unicode extends CharsetRecognizer {
}
static class CharsetRecog_UTF_16_LE extends CharsetRecog_Unicode {
@Override
String getName() {
return "UTF-16LE";
}
@Override
CharsetMatch match(CharsetDetector det) {
byte[] input = det.fRawInput;
int confidence = 10;
@ -114,8 +120,10 @@ abstract class CharsetRecog_Unicode extends CharsetRecognizer {
static abstract class CharsetRecog_UTF_32 extends CharsetRecog_Unicode {
abstract int getChar(byte[] input, int index);
@Override
abstract String getName();
@Override
CharsetMatch match(CharsetDetector det) {
byte[] input = det.fRawInput;
int limit = (det.fRawLength / 4) * 4;
@ -162,11 +170,13 @@ abstract class CharsetRecog_Unicode extends CharsetRecognizer {
}
static class CharsetRecog_UTF_32_BE extends CharsetRecog_UTF_32 {
@Override
int getChar(byte[] input, int index) {
return (input[index + 0] & 0xFF) << 24 | (input[index + 1] & 0xFF) << 16 |
(input[index + 2] & 0xFF) << 8 | (input[index + 3] & 0xFF);
}
@Override
String getName() {
return "UTF-32BE";
}
@ -174,11 +184,13 @@ abstract class CharsetRecog_Unicode extends CharsetRecognizer {
static class CharsetRecog_UTF_32_LE extends CharsetRecog_UTF_32 {
@Override
int getChar(byte[] input, int index) {
return (input[index + 3] & 0xFF) << 24 | (input[index + 2] & 0xFF) << 16 |
(input[index + 1] & 0xFF) << 8 | (input[index + 0] & 0xFF);
}
@Override
String getName() {
return "UTF-32LE";
}

32
app/src/main/java/io/legado/app/lib/icu4j/CharsetRecog_mbcs.java
Unescape View File

@ -1,4 +1,5 @@
/* GENERATED SOURCE. DO NOT MODIFY. */
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
****************************************************************************
* Copyright (C) 2005-2012, International Business Machines Corporation and *
@ -8,7 +9,6 @@
*/
package io.legado.app.lib.icu4j;
import java.util.Arrays;
/**
@ -30,6 +30,7 @@ abstract class CharsetRecog_mbcs extends CharsetRecognizer {
*
* @return the charset name.
*/
@Override
abstract String getName();
@ -142,14 +143,12 @@ abstract class CharsetRecog_mbcs extends CharsetRecognizer {
//
static class iteratedChar {
int charValue = 0; // 1-4 bytes from the raw input data
int index = 0;
int nextIndex = 0;
boolean error = false;
boolean done = false;
void reset() {
charValue = 0;
index = -1;
nextIndex = 0;
error = false;
done = false;
@ -160,7 +159,7 @@ abstract class CharsetRecog_mbcs extends CharsetRecognizer {
done = true;
return -1;
}
int byteValue = (int) det.fRawInput[nextIndex++] & 0x00ff;
int byteValue = det.fRawInput[nextIndex++] & 0x00ff;
return byteValue;
}
}
@ -196,8 +195,8 @@ abstract class CharsetRecog_mbcs extends CharsetRecognizer {
0x8343, 0x834e, 0x834f, 0x8358, 0x835e, 0x8362, 0x8367, 0x8375, 0x8376, 0x8389,
0x838a, 0x838b, 0x838d, 0x8393, 0x8e96, 0x93fa, 0x95aa};
@Override
boolean nextChar(iteratedChar it, CharsetDetector det) {
it.index = it.nextIndex;
it.error = false;
int firstByte;
firstByte = it.charValue = it.nextByte(det);
@ -221,15 +220,18 @@ abstract class CharsetRecog_mbcs extends CharsetRecognizer {
return true;
}
@Override
CharsetMatch match(CharsetDetector det) {
int confidence = match(det, commonChars);
return confidence == 0 ? null : new CharsetMatch(det, this, confidence);
}
@Override
String getName() {
return "Shift_JIS";
}
@Override
public String getLanguage() {
return "ja";
}
@ -257,8 +259,8 @@ abstract class CharsetRecog_mbcs extends CharsetRecognizer {
0xb5a5, 0xb5bd, 0xb5d0, 0xb5d8, 0xb671, 0xb7ed, 0xb867, 0xb944, 0xbad8, 0xbb44,
0xbba1, 0xbdd1, 0xc2c4, 0xc3b9, 0xc440, 0xc45f};
@Override
boolean nextChar(iteratedChar it, CharsetDetector det) {
it.index = it.nextIndex;
it.error = false;
int firstByte;
firstByte = it.charValue = it.nextByte(det);
@ -285,16 +287,19 @@ abstract class CharsetRecog_mbcs extends CharsetRecognizer {
return true;
}
@Override
CharsetMatch match(CharsetDetector det) {
int confidence = match(det, commonChars);
return confidence == 0 ? null : new CharsetMatch(det, this, confidence);
}
@Override
String getName() {
return "Big5";
}
@Override
public String getLanguage() {
return "zh";
}
@ -314,8 +319,8 @@ abstract class CharsetRecog_mbcs extends CharsetRecognizer {
* Character "value" is simply the raw bytes that make up the character
* packed into an int.
*/
@Override
boolean nextChar(iteratedChar it, CharsetDetector det) {
it.index = it.nextIndex;
it.error = false;
int firstByte = 0;
int secondByte = 0;
@ -392,15 +397,18 @@ abstract class CharsetRecog_mbcs extends CharsetRecognizer {
0xa5e5, 0xa5e9, 0xa5ea, 0xa5eb, 0xa5ec, 0xa5ed, 0xa5f3, 0xb8a9, 0xb9d4, 0xbaee,
0xbbc8, 0xbef0, 0xbfb7, 0xc4ea, 0xc6fc, 0xc7bd, 0xcab8, 0xcaf3, 0xcbdc, 0xcdd1};
@Override
String getName() {
return "EUC-JP";
}
@Override
CharsetMatch match(CharsetDetector det) {
int confidence = match(det, commonChars);
return confidence == 0 ? null : new CharsetMatch(det, this, confidence);
}
@Override
public String getLanguage() {
return "ja";
}
@ -426,15 +434,18 @@ abstract class CharsetRecog_mbcs extends CharsetRecognizer {
0xc0da, 0xc0e5, 0xc0fb, 0xc0fc, 0xc1a4, 0xc1a6, 0xc1b6, 0xc1d6, 0xc1df, 0xc1f6,
0xc1f8, 0xc4a1, 0xc5cd, 0xc6ae, 0xc7cf, 0xc7d1, 0xc7d2, 0xc7d8, 0xc7e5, 0xc8ad};
@Override
String getName() {
return "EUC-KR";
}
@Override
CharsetMatch match(CharsetDetector det) {
int confidence = match(det, commonChars);
return confidence == 0 ? null : new CharsetMatch(det, this, confidence);
}
@Override
public String getLanguage() {
return "ko";
}
@ -452,8 +463,8 @@ abstract class CharsetRecog_mbcs extends CharsetRecognizer {
* Character "value" is simply the raw bytes that make up the character
* packed into an int.
*/
@Override
boolean nextChar(iteratedChar it, CharsetDetector det) {
it.index = it.nextIndex;
it.error = false;
int firstByte = 0;
int secondByte = 0;
@ -522,15 +533,18 @@ abstract class CharsetRecog_mbcs extends CharsetRecognizer {
0xd2b5, 0xd2bb, 0xd2d4, 0xd3c3, 0xd3d0, 0xd3fd, 0xd4c2, 0xd4da, 0xd5e2, 0xd6d0};
@Override
String getName() {
return "GB18030";
}
@Override
CharsetMatch match(CharsetDetector det) {
int confidence = match(det, commonChars);
return confidence == 0 ? null : new CharsetMatch(det, this, confidence);
}
@Override
public String getLanguage() {
return "zh";
}

47
app/src/main/java/io/legado/app/lib/icu4j/CharsetRecog_sbcs.java
Unescape View File

@ -1,4 +1,5 @@
/* GENERATED SOURCE. DO NOT MODIFY. */
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/*
****************************************************************************
* Copyright (C) 2005-2013, International Business Machines Corporation and *
@ -9,7 +10,6 @@
package io.legado.app.lib.icu4j;
/**
* This class recognizes single-byte encodings. Because the encoding scheme is so
* simple, language statistics are used to do the matching.
@ -19,6 +19,7 @@ abstract class CharsetRecog_sbcs extends CharsetRecognizer {
/* (non-Javadoc)
* @see com.ibm.icu.text.CharsetRecognizer#getName()
*/
@Override
abstract String getName();
static class NGramParser {
@ -219,6 +220,7 @@ abstract class CharsetRecog_sbcs extends CharsetRecognizer {
return next;
}
@Override
protected void parseCharacters(CharsetDetector det) {
int b;
boolean ignoreSpace = false;
@ -406,6 +408,7 @@ abstract class CharsetRecog_sbcs extends CharsetRecognizer {
};
@Override
public CharsetMatch match(CharsetDetector det) {
String name = det.fC1Bytes ? "windows-1252" : "ISO-8859-1";
int bestConfidenceSoFar = -1;
@ -421,6 +424,7 @@ abstract class CharsetRecog_sbcs extends CharsetRecognizer {
}
@Override
public String getName() {
return "ISO-8859-1";
}
@ -498,6 +502,7 @@ abstract class CharsetRecog_sbcs extends CharsetRecognizer {
})
};
@Override
public CharsetMatch match(CharsetDetector det) {
String name = det.fC1Bytes ? "windows-1250" : "ISO-8859-2";
int bestConfidenceSoFar = -1;
@ -512,6 +517,7 @@ abstract class CharsetRecog_sbcs extends CharsetRecognizer {
return bestConfidenceSoFar <= 0 ? null : new CharsetMatch(det, this, bestConfidenceSoFar, name, lang);
}
@Override
public String getName() {
return "ISO-8859-2";
}
@ -555,6 +561,7 @@ abstract class CharsetRecog_sbcs extends CharsetRecognizer {
(byte) 0xF8, (byte) 0xF9, (byte) 0xFA, (byte) 0xFB, (byte) 0xFC, (byte) 0x20, (byte) 0xFE, (byte) 0xFF,
};
@Override
public String getName() {
return "ISO-8859-5";
}
@ -568,10 +575,12 @@ abstract class CharsetRecog_sbcs extends CharsetRecognizer {
0xDFDEDB, 0xDFE0D5, 0xDFE0D8, 0xDFE0DE, 0xE0D0D2, 0xE0D5D4, 0xE1E2D0, 0xE1E2D2, 0xE1E2D8, 0xE1EF20, 0xE2D5DB, 0xE2DE20, 0xE2DEE0, 0xE2EC20, 0xE7E2DE, 0xEBE520,
};
@Override
public String getLanguage() {
return "ru";
}
@Override
public CharsetMatch match(CharsetDetector det) {
int confidence = match(det, ngrams, byteMap);
return confidence == 0 ? null : new CharsetMatch(det, this, confidence);
@ -614,6 +623,7 @@ abstract class CharsetRecog_sbcs extends CharsetRecognizer {
(byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
};
@Override
public String getName() {
return "ISO-8859-6";
}
@ -627,10 +637,12 @@ abstract class CharsetRecog_sbcs extends CharsetRecognizer {
0xE520C7, 0xE5C720, 0xE5C920, 0xE5E620, 0xE620C7, 0xE720C7, 0xE7C720, 0xE8C7E4, 0xE8E620, 0xE920C7, 0xEA20C7, 0xEA20E5, 0xEA20E8, 0xEAC920, 0xEAD120, 0xEAE620,
};
@Override
public String getLanguage() {
return "ar";
}
@Override
public CharsetMatch match(CharsetDetector det) {
int confidence = match(det, ngrams, byteMap);
return confidence == 0 ? null : new CharsetMatch(det, this, confidence);
@ -673,6 +685,7 @@ abstract class CharsetRecog_sbcs extends CharsetRecognizer {
(byte) 0xF8, (byte) 0xF9, (byte) 0xFA, (byte) 0xFB, (byte) 0xFC, (byte) 0xFD, (byte) 0xFE, (byte) 0x20,
};
@Override
public String getName() {
return "ISO-8859-7";
}
@ -686,10 +699,12 @@ abstract class CharsetRecog_sbcs extends CharsetRecognizer {
0xF220EA, 0xF220F0, 0xF220F4, 0xF3E520, 0xF3E720, 0xF3F4EF, 0xF4E120, 0xF4E1E9, 0xF4E7ED, 0xF4E7F2, 0xF4E9EA, 0xF4EF20, 0xF4EFF5, 0xF4F9ED, 0xF9ED20, 0xFEED20,
};
@Override
public String getLanguage() {
return "el";
}
@Override
public CharsetMatch match(CharsetDetector det) {
String name = det.fC1Bytes ? "windows-1253" : "ISO-8859-7";
int confidence = match(det, ngrams, byteMap);
@ -733,6 +748,7 @@ abstract class CharsetRecog_sbcs extends CharsetRecognizer {
(byte) 0xF8, (byte) 0xF9, (byte) 0xFA, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20,
};
@Override
public String getName() {
return "ISO-8859-8";
}
@ -746,14 +762,17 @@ abstract class CharsetRecog_sbcs extends CharsetRecognizer {
0xED20F9, 0xEEE420, 0xEF20E4, 0xF0E420, 0xF0E920, 0xF0E9ED, 0xF2EC20, 0xF820E4, 0xF8E9ED, 0xF9EC20, 0xFA20E0, 0xFA20E1, 0xFA20E4, 0xFA20EC, 0xFA20EE, 0xFA20F9,
};
@Override
public String getName() {
return "ISO-8859-8-I";
}
@Override
public String getLanguage() {
return "he";
}
@Override
public CharsetMatch match(CharsetDetector det) {
String name = det.fC1Bytes ? "windows-1255" : "ISO-8859-8-I";
int confidence = match(det, ngrams, byteMap);
@ -769,10 +788,12 @@ abstract class CharsetRecog_sbcs extends CharsetRecognizer {
0xEDE9E9, 0xEDE9F0, 0xEDE9F8, 0xEE20E4, 0xEE20ED, 0xEE20FA, 0xEEE120, 0xEEE420, 0xF2E420, 0xF920E4, 0xF920ED, 0xF920FA, 0xF9E420, 0xFAE020, 0xFAE420, 0xFAE5E9,
};
@Override
public String getLanguage() {
return "he";
}
@Override
public CharsetMatch match(CharsetDetector det) {
String name = det.fC1Bytes ? "windows-1255" : "ISO-8859-8";
int confidence = match(det, ngrams, byteMap);
@ -817,6 +838,7 @@ abstract class CharsetRecog_sbcs extends CharsetRecognizer {
(byte) 0xF8, (byte) 0xF9, (byte) 0xFA, (byte) 0xFB, (byte) 0xFC, (byte) 0xFD, (byte) 0xFE, (byte) 0xFF,
};
@Override
public String getName() {
return "ISO-8859-9";
}
@ -830,10 +852,12 @@ abstract class CharsetRecog_sbcs extends CharsetRecognizer {
0x6E206B, 0x6E6461, 0x6E6465, 0x6E6520, 0x6E6920, 0x6E696E, 0x6EFD20, 0x72696E, 0x72FD6E, 0x766520, 0x796120, 0x796F72, 0xFD6E20, 0xFD6E64, 0xFD6EFD, 0xFDF0FD,
};
@Override
public String getLanguage() {
return "tr";
}
@Override
public CharsetMatch match(CharsetDetector det) {
String name = det.fC1Bytes ? "windows-1254" : "ISO-8859-9";
int confidence = match(det, ngrams, byteMap);
@ -884,14 +908,17 @@ abstract class CharsetRecog_sbcs extends CharsetRecognizer {
(byte) 0xF8, (byte) 0xF9, (byte) 0xFA, (byte) 0xFB, (byte) 0xFC, (byte) 0xFD, (byte) 0xFE, (byte) 0xFF,
};
@Override
public String getName() {
return "windows-1251";
}
@Override
public String getLanguage() {
return "ru";
}
@Override
public CharsetMatch match(CharsetDetector det) {
int confidence = match(det, ngrams, byteMap);
return confidence == 0 ? null : new CharsetMatch(det, this, confidence);
@ -941,14 +968,17 @@ abstract class CharsetRecog_sbcs extends CharsetRecognizer {
(byte) 0x20, (byte) 0xF9, (byte) 0x20, (byte) 0xFB, (byte) 0xFC, (byte) 0x20, (byte) 0x20, (byte) 0xFF,
};
@Override
public String getName() {
return "windows-1256";
}
@Override
public String getLanguage() {
return "ar";
}
@Override
public CharsetMatch match(CharsetDetector det) {
int confidence = match(det, ngrams, byteMap);
return confidence == 0 ? null : new CharsetMatch(det, this, confidence);
@ -998,14 +1028,17 @@ abstract class CharsetRecog_sbcs extends CharsetRecognizer {
(byte) 0xD8, (byte) 0xD9, (byte) 0xDA, (byte) 0xDB, (byte) 0xDC, (byte) 0xDD, (byte) 0xDE, (byte) 0xDF,
};
@Override
public String getName() {
return "KOI8-R";
}
@Override
public String getLanguage() {
return "ru";
}
@Override
public CharsetMatch match(CharsetDetector det) {
int confidence = match(det, ngrams, byteMap);
return confidence == 0 ? null : new CharsetMatch(det, this, confidence);
@ -1033,12 +1066,14 @@ abstract class CharsetRecog_sbcs extends CharsetRecognizer {
/* F- */ (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40, (byte) 0x40,
};
@Override
public String getLanguage() {
return "he";
}
}
static class CharsetRecog_IBM424_he_rtl extends CharsetRecog_IBM424_he {
@Override
public String getName() {
return "IBM424_rtl";
}
@ -1050,6 +1085,7 @@ abstract class CharsetRecog_sbcs extends CharsetRecognizer {
0x554069, 0x564540, 0x574045, 0x584540, 0x585140, 0x585155, 0x625440, 0x684045, 0x685155, 0x695440, 0x714041, 0x714042, 0x714045, 0x714054, 0x714056, 0x714069,
};
@Override
public CharsetMatch match(CharsetDetector det) {
int confidence = match(det, ngrams, byteMap, (byte) 0x40);
return confidence == 0 ? null : new CharsetMatch(det, this, confidence);
@ -1057,6 +1093,7 @@ abstract class CharsetRecog_sbcs extends CharsetRecognizer {
}
static class CharsetRecog_IBM424_he_ltr extends CharsetRecog_IBM424_he {
@Override
public String getName() {
return "IBM424_ltr";
}
@ -1069,6 +1106,7 @@ abstract class CharsetRecog_sbcs extends CharsetRecognizer {
};
@Override
public CharsetMatch match(CharsetDetector det) {
int confidence = match(det, ngrams, byteMap, (byte) 0x40);
return confidence == 0 ? null : new CharsetMatch(det, this, confidence);
@ -1098,6 +1136,7 @@ abstract class CharsetRecog_sbcs extends CharsetRecognizer {
};
@Override
public String getLanguage() {
return "ar";
}
@ -1112,10 +1151,12 @@ abstract class CharsetRecog_sbcs extends CharsetRecognizer {
0xBB4056, 0xBB5640, 0xBB6240, 0xBBBD40, 0xBD4056, 0xBF4056, 0xBF5640, 0xCF56B1, 0xCFBD40, 0xDA4056, 0xDC4056, 0xDC40BB, 0xDC40CF, 0xDC6240, 0xDC7540, 0xDCBD40,
};
@Override
public String getName() {
return "IBM420_rtl";
}
@Override
public CharsetMatch match(CharsetDetector det) {
int confidence = matchIBM420(det, ngrams, byteMap, (byte) 0x40);
return confidence == 0 ? null : new CharsetMatch(det, this, confidence);
@ -1131,10 +1172,12 @@ abstract class CharsetRecog_sbcs extends CharsetRecognizer {
0xB14062, 0xB15640, 0xB156CF, 0xB19A40, 0xB1B140, 0xBB4062, 0xBB40DC, 0xBBB156, 0xBD5640, 0xBDBB40, 0xCF4062, 0xCF40DC, 0xCFB156, 0xDAB19A, 0xDCAB40, 0xDCB156
};
@Override
public String getName() {
return "IBM420_ltr";
}
@Override
public CharsetMatch match(CharsetDetector det) {
int confidence = matchIBM420(det, ngrams, byteMap, (byte) 0x40);
return confidence == 0 ? null : new CharsetMatch(det, this, confidence);

14
app/src/main/java/io/legado/app/lib/icu4j/CharsetRecognizer.java
Unescape View File

@ -1,4 +1,5 @@
/* GENERATED SOURCE. DO NOT MODIFY. */
// © 2016 and later: Unicode, Inc. and others.
// License & terms of use: http://www.unicode.org/copyright.html
/**
* ******************************************************************************
* Copyright (C) 2005-2012, International Business Machines Corporation and *
@ -7,7 +8,6 @@
*/
package io.legado.app.lib.icu4j;
/**
* Abstract class for recognizing a single charset.
* Part of the implementation of ICU's CharsetDetector.
@ -25,14 +25,12 @@ package io.legado.app.lib.icu4j;
abstract class CharsetRecognizer {
/**
* Get the IANA name of this charset.
*
* @return the charset name.
*/
abstract String getName();
/**
* Get the ISO language code for this charset.
*
* @return the language code, or <code>null</code> if the language cannot be determined.
*/
public String getLanguage() {
@ -41,12 +39,12 @@ abstract class CharsetRecognizer {
/**
* Test the match of this charset with the input text data
* which is obtained via the CharsetDetector object.
* which is obtained via the CharsetDetector object.
*
* @param det The CharsetDetector, which contains the input text
* to be checked for being in this charset.
* @param det The CharsetDetector, which contains the input text
* to be checked for being in this charset.
* @return A CharsetMatch object containing details of match
* with this charset, or null if there was no match.
* with this charset, or null if there was no match.
*/
abstract CharsetMatch match(CharsetDetector det);

Write Preview
Loading…
Cancel
Save