|
|
|
@ -29,7 +29,7 @@ abstract class CharsetRecog_sbcs extends CharsetRecognizer { |
|
|
|
|
protected int byteIndex = 0; |
|
|
|
|
private int ngram = 0; |
|
|
|
|
|
|
|
|
|
private int[] ngramList; |
|
|
|
|
private final int[] ngramList; |
|
|
|
|
protected byte[] byteMap; |
|
|
|
|
|
|
|
|
|
private int ngramCount; |
|
|
|
@ -315,7 +315,7 @@ abstract class CharsetRecog_sbcs extends CharsetRecognizer { |
|
|
|
|
}; |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
private static NGramsPlusLang[] ngrams_8859_1 = new NGramsPlusLang[]{ |
|
|
|
|
private static final NGramsPlusLang[] ngrams_8859_1 = new NGramsPlusLang[]{ |
|
|
|
|
new NGramsPlusLang( |
|
|
|
|
"da", |
|
|
|
|
new int[]{ |
|
|
|
@ -467,7 +467,7 @@ abstract class CharsetRecog_sbcs extends CharsetRecognizer { |
|
|
|
|
(byte) 0xF8, (byte) 0xF9, (byte) 0xFA, (byte) 0xFB, (byte) 0xFC, (byte) 0xFD, (byte) 0xFE, (byte) 0x20, |
|
|
|
|
}; |
|
|
|
|
|
|
|
|
|
private static NGramsPlusLang[] ngrams_8859_2 = new NGramsPlusLang[]{ |
|
|
|
|
private static final NGramsPlusLang[] ngrams_8859_2 = new NGramsPlusLang[]{ |
|
|
|
|
new NGramsPlusLang( |
|
|
|
|
"cs", |
|
|
|
|
new int[]{ |
|
|
|
@ -568,7 +568,7 @@ abstract class CharsetRecog_sbcs extends CharsetRecognizer { |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
static class CharsetRecog_8859_5_ru extends CharsetRecog_8859_5 { |
|
|
|
|
private static int[] ngrams = { |
|
|
|
|
private static final int[] ngrams = { |
|
|
|
|
0x20D220, 0x20D2DE, 0x20D4DE, 0x20D7D0, 0x20D820, 0x20DAD0, 0x20DADE, 0x20DDD0, 0x20DDD5, 0x20DED1, 0x20DFDE, 0x20DFE0, 0x20E0D0, 0x20E1DE, 0x20E1E2, 0x20E2DE, |
|
|
|
|
0x20E7E2, 0x20EDE2, 0xD0DDD8, 0xD0E2EC, 0xD3DE20, 0xD5DBEC, 0xD5DDD8, 0xD5E1E2, 0xD5E220, 0xD820DF, 0xD8D520, 0xD8D820, 0xD8EF20, 0xDBD5DD, 0xDBD820, 0xDBECDD, |
|
|
|
|
0xDDD020, 0xDDD520, 0xDDD8D5, 0xDDD8EF, 0xDDDE20, 0xDDDED2, 0xDE20D2, 0xDE20DF, 0xDE20E1, 0xDED220, 0xDED2D0, 0xDED3DE, 0xDED920, 0xDEDBEC, 0xDEDC20, 0xDEE1E2, |
|
|
|
@ -630,7 +630,7 @@ abstract class CharsetRecog_sbcs extends CharsetRecognizer { |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
static class CharsetRecog_8859_6_ar extends CharsetRecog_8859_6 { |
|
|
|
|
private static int[] ngrams = { |
|
|
|
|
private static final int[] ngrams = { |
|
|
|
|
0x20C7E4, 0x20C7E6, 0x20C8C7, 0x20D9E4, 0x20E1EA, 0x20E4E4, 0x20E5E6, 0x20E8C7, 0xC720C7, 0xC7C120, 0xC7CA20, 0xC7D120, 0xC7E420, 0xC7E4C3, 0xC7E4C7, 0xC7E4C8, |
|
|
|
|
0xC7E4CA, 0xC7E4CC, 0xC7E4CD, 0xC7E4CF, 0xC7E4D3, 0xC7E4D9, 0xC7E4E2, 0xC7E4E5, 0xC7E4E8, 0xC7E4EA, 0xC7E520, 0xC7E620, 0xC7E6CA, 0xC820C7, 0xC920C7, 0xC920E1, |
|
|
|
|
0xC920E4, 0xC920E5, 0xC920E8, 0xCA20C7, 0xCF20C7, 0xCFC920, 0xD120C7, 0xD1C920, 0xD320C7, 0xD920C7, 0xD9E4E9, 0xE1EA20, 0xE420C7, 0xE4C920, 0xE4E920, 0xE4EA20, |
|
|
|
@ -692,7 +692,7 @@ abstract class CharsetRecog_sbcs extends CharsetRecognizer { |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
static class CharsetRecog_8859_7_el extends CharsetRecog_8859_7 { |
|
|
|
|
private static int[] ngrams = { |
|
|
|
|
private static final int[] ngrams = { |
|
|
|
|
0x20E1ED, 0x20E1F0, 0x20E3E9, 0x20E4E9, 0x20E5F0, 0x20E720, 0x20EAE1, 0x20ECE5, 0x20EDE1, 0x20EF20, 0x20F0E1, 0x20F0EF, 0x20F0F1, 0x20F3F4, 0x20F3F5, 0x20F4E7, |
|
|
|
|
0x20F4EF, 0xDFE120, 0xE120E1, 0xE120F4, 0xE1E920, 0xE1ED20, 0xE1F0FC, 0xE1F220, 0xE3E9E1, 0xE5E920, 0xE5F220, 0xE720F4, 0xE7ED20, 0xE7F220, 0xE920F4, 0xE9E120, |
|
|
|
|
0xE9EADE, 0xE9F220, 0xEAE1E9, 0xEAE1F4, 0xECE520, 0xED20E1, 0xED20E5, 0xED20F0, 0xEDE120, 0xEFF220, 0xEFF520, 0xF0EFF5, 0xF0F1EF, 0xF0FC20, 0xF220E1, 0xF220E5, |
|
|
|
@ -755,7 +755,7 @@ abstract class CharsetRecog_sbcs extends CharsetRecognizer { |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
static class CharsetRecog_8859_8_I_he extends CharsetRecog_8859_8 { |
|
|
|
|
private static int[] ngrams = { |
|
|
|
|
private static final int[] ngrams = { |
|
|
|
|
0x20E0E5, 0x20E0E7, 0x20E0E9, 0x20E0FA, 0x20E1E9, 0x20E1EE, 0x20E4E0, 0x20E4E5, 0x20E4E9, 0x20E4EE, 0x20E4F2, 0x20E4F9, 0x20E4FA, 0x20ECE0, 0x20ECE4, 0x20EEE0, |
|
|
|
|
0x20F2EC, 0x20F9EC, 0xE0FA20, 0xE420E0, 0xE420E1, 0xE420E4, 0xE420EC, 0xE420EE, 0xE420F9, 0xE4E5E0, 0xE5E020, 0xE5ED20, 0xE5EF20, 0xE5F820, 0xE5FA20, 0xE920E4, |
|
|
|
|
0xE9E420, 0xE9E5FA, 0xE9E9ED, 0xE9ED20, 0xE9EF20, 0xE9F820, 0xE9FA20, 0xEC20E0, 0xEC20E4, 0xECE020, 0xECE420, 0xED20E0, 0xED20E1, 0xED20E4, 0xED20EC, 0xED20EE, |
|
|
|
@ -781,7 +781,7 @@ abstract class CharsetRecog_sbcs extends CharsetRecognizer { |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
static class CharsetRecog_8859_8_he extends CharsetRecog_8859_8 { |
|
|
|
|
private static int[] ngrams = { |
|
|
|
|
private static final int[] ngrams = { |
|
|
|
|
0x20E0E5, 0x20E0EC, 0x20E4E9, 0x20E4EC, 0x20E4EE, 0x20E4F0, 0x20E9F0, 0x20ECF2, 0x20ECF9, 0x20EDE5, 0x20EDE9, 0x20EFE5, 0x20EFE9, 0x20F8E5, 0x20F8E9, 0x20FAE0, |
|
|
|
|
0x20FAE5, 0x20FAE9, 0xE020E4, 0xE020EC, 0xE020ED, 0xE020FA, 0xE0E420, 0xE0E5E4, 0xE0EC20, 0xE0EE20, 0xE120E4, 0xE120ED, 0xE120FA, 0xE420E4, 0xE420E9, 0xE420EC, |
|
|
|
|
0xE420ED, 0xE420EF, 0xE420F8, 0xE420FA, 0xE4EC20, 0xE5E020, 0xE5E420, 0xE7E020, 0xE9E020, 0xE9E120, 0xE9E420, 0xEC20E4, 0xEC20ED, 0xEC20FA, 0xECF220, 0xECF920, |
|
|
|
@ -845,7 +845,7 @@ abstract class CharsetRecog_sbcs extends CharsetRecognizer { |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
static class CharsetRecog_8859_9_tr extends CharsetRecog_8859_9 { |
|
|
|
|
private static int[] ngrams = { |
|
|
|
|
private static final int[] ngrams = { |
|
|
|
|
0x206261, 0x206269, 0x206275, 0x206461, 0x206465, 0x206765, 0x206861, 0x20696C, 0x206B61, 0x206B6F, 0x206D61, 0x206F6C, 0x207361, 0x207461, 0x207665, 0x207961, |
|
|
|
|
0x612062, 0x616B20, 0x616C61, 0x616D61, 0x616E20, 0x616EFD, 0x617220, 0x617261, 0x6172FD, 0x6173FD, 0x617961, 0x626972, 0x646120, 0x646520, 0x646920, 0x652062, |
|
|
|
|
0x65206B, 0x656469, 0x656E20, 0x657220, 0x657269, 0x657369, 0x696C65, 0x696E20, 0x696E69, 0x697220, 0x6C616E, 0x6C6172, 0x6C6520, 0x6C6572, 0x6E2061, 0x6E2062, |
|
|
|
@ -866,14 +866,14 @@ abstract class CharsetRecog_sbcs extends CharsetRecognizer { |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
static class CharsetRecog_windows_1251 extends CharsetRecog_sbcs { |
|
|
|
|
private static int[] ngrams = { |
|
|
|
|
private static final int[] ngrams = { |
|
|
|
|
0x20E220, 0x20E2EE, 0x20E4EE, 0x20E7E0, 0x20E820, 0x20EAE0, 0x20EAEE, 0x20EDE0, 0x20EDE5, 0x20EEE1, 0x20EFEE, 0x20EFF0, 0x20F0E0, 0x20F1EE, 0x20F1F2, 0x20F2EE, |
|
|
|
|
0x20F7F2, 0x20FDF2, 0xE0EDE8, 0xE0F2FC, 0xE3EE20, 0xE5EBFC, 0xE5EDE8, 0xE5F1F2, 0xE5F220, 0xE820EF, 0xE8E520, 0xE8E820, 0xE8FF20, 0xEBE5ED, 0xEBE820, 0xEBFCED, |
|
|
|
|
0xEDE020, 0xEDE520, 0xEDE8E5, 0xEDE8FF, 0xEDEE20, 0xEDEEE2, 0xEE20E2, 0xEE20EF, 0xEE20F1, 0xEEE220, 0xEEE2E0, 0xEEE3EE, 0xEEE920, 0xEEEBFC, 0xEEEC20, 0xEEF1F2, |
|
|
|
|
0xEFEEEB, 0xEFF0E5, 0xEFF0E8, 0xEFF0EE, 0xF0E0E2, 0xF0E5E4, 0xF1F2E0, 0xF1F2E2, 0xF1F2E8, 0xF1FF20, 0xF2E5EB, 0xF2EE20, 0xF2EEF0, 0xF2FC20, 0xF7F2EE, 0xFBF520, |
|
|
|
|
}; |
|
|
|
|
|
|
|
|
|
private static byte[] byteMap = { |
|
|
|
|
private static final byte[] byteMap = { |
|
|
|
|
(byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, |
|
|
|
|
(byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, |
|
|
|
|
(byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, |
|
|
|
@ -926,14 +926,14 @@ abstract class CharsetRecog_sbcs extends CharsetRecognizer { |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
static class CharsetRecog_windows_1256 extends CharsetRecog_sbcs { |
|
|
|
|
private static int[] ngrams = { |
|
|
|
|
private static final int[] ngrams = { |
|
|
|
|
0x20C7E1, 0x20C7E4, 0x20C8C7, 0x20DAE1, 0x20DDED, 0x20E1E1, 0x20E3E4, 0x20E6C7, 0xC720C7, 0xC7C120, 0xC7CA20, 0xC7D120, 0xC7E120, 0xC7E1C3, 0xC7E1C7, 0xC7E1C8, |
|
|
|
|
0xC7E1CA, 0xC7E1CC, 0xC7E1CD, 0xC7E1CF, 0xC7E1D3, 0xC7E1DA, 0xC7E1DE, 0xC7E1E3, 0xC7E1E6, 0xC7E1ED, 0xC7E320, 0xC7E420, 0xC7E4CA, 0xC820C7, 0xC920C7, 0xC920DD, |
|
|
|
|
0xC920E1, 0xC920E3, 0xC920E6, 0xCA20C7, 0xCF20C7, 0xCFC920, 0xD120C7, 0xD1C920, 0xD320C7, 0xDA20C7, 0xDAE1EC, 0xDDED20, 0xE120C7, 0xE1C920, 0xE1EC20, 0xE1ED20, |
|
|
|
|
0xE320C7, 0xE3C720, 0xE3C920, 0xE3E420, 0xE420C7, 0xE520C7, 0xE5C720, 0xE6C7E1, 0xE6E420, 0xEC20C7, 0xED20C7, 0xED20E3, 0xED20E6, 0xEDC920, 0xEDD120, 0xEDE420, |
|
|
|
|
}; |
|
|
|
|
|
|
|
|
|
private static byte[] byteMap = { |
|
|
|
|
private static final byte[] byteMap = { |
|
|
|
|
(byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, |
|
|
|
|
(byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, |
|
|
|
|
(byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, |
|
|
|
@ -986,14 +986,14 @@ abstract class CharsetRecog_sbcs extends CharsetRecognizer { |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
static class CharsetRecog_KOI8_R extends CharsetRecog_sbcs { |
|
|
|
|
private static int[] ngrams = { |
|
|
|
|
private static final int[] ngrams = { |
|
|
|
|
0x20C4CF, 0x20C920, 0x20CBC1, 0x20CBCF, 0x20CEC1, 0x20CEC5, 0x20CFC2, 0x20D0CF, 0x20D0D2, 0x20D2C1, 0x20D3CF, 0x20D3D4, 0x20D4CF, 0x20D720, 0x20D7CF, 0x20DAC1, |
|
|
|
|
0x20DCD4, 0x20DED4, 0xC1CEC9, 0xC1D4D8, 0xC5CCD8, 0xC5CEC9, 0xC5D3D4, 0xC5D420, 0xC7CF20, 0xC920D0, 0xC9C520, 0xC9C920, 0xC9D120, 0xCCC5CE, 0xCCC920, 0xCCD8CE, |
|
|
|
|
0xCEC120, 0xCEC520, 0xCEC9C5, 0xCEC9D1, 0xCECF20, 0xCECFD7, 0xCF20D0, 0xCF20D3, 0xCF20D7, 0xCFC7CF, 0xCFCA20, 0xCFCCD8, 0xCFCD20, 0xCFD3D4, 0xCFD720, 0xCFD7C1, |
|
|
|
|
0xD0CFCC, 0xD0D2C5, 0xD0D2C9, 0xD0D2CF, 0xD2C1D7, 0xD2C5C4, 0xD3D120, 0xD3D4C1, 0xD3D4C9, 0xD3D4D7, 0xD4C5CC, 0xD4CF20, 0xD4CFD2, 0xD4D820, 0xD9C820, 0xDED4CF, |
|
|
|
|
}; |
|
|
|
|
|
|
|
|
|
private static byte[] byteMap = { |
|
|
|
|
private static final byte[] byteMap = { |
|
|
|
|
(byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, |
|
|
|
|
(byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, |
|
|
|
|
(byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, (byte) 0x20, |
|
|
|
@ -1078,7 +1078,7 @@ abstract class CharsetRecog_sbcs extends CharsetRecognizer { |
|
|
|
|
return "IBM424_rtl"; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
private static int[] ngrams = { |
|
|
|
|
private static final int[] ngrams = { |
|
|
|
|
0x404146, 0x404148, 0x404151, 0x404171, 0x404251, 0x404256, 0x404541, 0x404546, 0x404551, 0x404556, 0x404562, 0x404569, 0x404571, 0x405441, 0x405445, 0x405641, |
|
|
|
|
0x406254, 0x406954, 0x417140, 0x454041, 0x454042, 0x454045, 0x454054, 0x454056, 0x454069, 0x454641, 0x464140, 0x465540, 0x465740, 0x466840, 0x467140, 0x514045, |
|
|
|
|
0x514540, 0x514671, 0x515155, 0x515540, 0x515740, 0x516840, 0x517140, 0x544041, 0x544045, 0x544140, 0x544540, 0x554041, 0x554042, 0x554045, 0x554054, 0x554056, |
|
|
|
@ -1098,7 +1098,7 @@ abstract class CharsetRecog_sbcs extends CharsetRecognizer { |
|
|
|
|
return "IBM424_ltr"; |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
private static int[] ngrams = { |
|
|
|
|
private static final int[] ngrams = { |
|
|
|
|
0x404146, 0x404154, 0x404551, 0x404554, 0x404556, 0x404558, 0x405158, 0x405462, 0x405469, 0x405546, 0x405551, 0x405746, 0x405751, 0x406846, 0x406851, 0x407141, |
|
|
|
|
0x407146, 0x407151, 0x414045, 0x414054, 0x414055, 0x414071, 0x414540, 0x414645, 0x415440, 0x415640, 0x424045, 0x424055, 0x424071, 0x454045, 0x454051, 0x454054, |
|
|
|
|
0x454055, 0x454057, 0x454068, 0x454071, 0x455440, 0x464140, 0x464540, 0x484140, 0x514140, 0x514240, 0x514540, 0x544045, 0x544055, 0x544071, 0x546240, 0x546940, |
|
|
|
@ -1144,7 +1144,7 @@ abstract class CharsetRecog_sbcs extends CharsetRecognizer { |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
static class CharsetRecog_IBM420_ar_rtl extends CharsetRecog_IBM420_ar { |
|
|
|
|
private static int[] ngrams = { |
|
|
|
|
private static final int[] ngrams = { |
|
|
|
|
0x4056B1, 0x4056BD, 0x405856, 0x409AB1, 0x40ABDC, 0x40B1B1, 0x40BBBD, 0x40CF56, 0x564056, 0x564640, 0x566340, 0x567540, 0x56B140, 0x56B149, 0x56B156, 0x56B158, |
|
|
|
|
0x56B163, 0x56B167, 0x56B169, 0x56B173, 0x56B178, 0x56B19A, 0x56B1AD, 0x56B1BB, 0x56B1CF, 0x56B1DC, 0x56BB40, 0x56BD40, 0x56BD63, 0x584056, 0x624056, 0x6240AB, |
|
|
|
|
0x6240B1, 0x6240BB, 0x6240CF, 0x634056, 0x734056, 0x736240, 0x754056, 0x756240, 0x784056, 0x9A4056, 0x9AB1DA, 0xABDC40, 0xB14056, 0xB16240, 0xB1DA40, 0xB1DC40, |
|
|
|
@ -1165,7 +1165,7 @@ abstract class CharsetRecog_sbcs extends CharsetRecognizer { |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
static class CharsetRecog_IBM420_ar_ltr extends CharsetRecog_IBM420_ar { |
|
|
|
|
private static int[] ngrams = { |
|
|
|
|
private static final int[] ngrams = { |
|
|
|
|
0x404656, 0x4056BB, 0x4056BF, 0x406273, 0x406275, 0x4062B1, 0x4062BB, 0x4062DC, 0x406356, 0x407556, 0x4075DC, 0x40B156, 0x40BB56, 0x40BD56, 0x40BDBB, 0x40BDCF, |
|
|
|
|
0x40BDDC, 0x40DAB1, 0x40DCAB, 0x40DCB1, 0x49B156, 0x564056, 0x564058, 0x564062, 0x564063, 0x564073, 0x564075, 0x564078, 0x56409A, 0x5640B1, 0x5640BB, 0x5640BD, |
|
|
|
|
0x5640BF, 0x5640DA, 0x5640DC, 0x565840, 0x56B156, 0x56CF40, 0x58B156, 0x63B156, 0x63BD56, 0x67B156, 0x69B156, 0x73B156, 0x78B156, 0x9AB156, 0xAB4062, 0xADB156, |
|
|
|
|