diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..f9f208ec --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,75 @@ +# This workflow will build a Java project with Maven +# For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven + +name: Java CI with Maven + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + test: + + strategy: + matrix: + java-version: [8, 11, 17, 21, 25] + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Set up JDK ${{ matrix.java-version }} + uses: actions/setup-java@v5 + with: + distribution: 'temurin' + java-version: ${{ matrix.java-version }} + cache: 'maven' + - name: Run tests + run: mvn -ntp test --file pom.xml + + version: + + needs: test + + outputs: + version: ${{ steps.version.outputs.version }} + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Set up JDK 11 + uses: actions/setup-java@v5 + with: + distribution: 'temurin' + java-version: 11 + cache: 'maven' + - name: version + run: | + echo version=`mvn help:evaluate -Dexpression=project.version -q -DforceStdout` >> $GITHUB_OUTPUT + + snapshot: + + needs: [test, version] + + runs-on: ubuntu-latest + + steps: + - name: checkout + uses: actions/checkout@v3 + - name: set up Java + uses: actions/setup-java@v2 + with: + distribution: 'temurin' + java-version: 11 + server-id: central + server-username: MAVEN_USERNAME + server-password: MAVEN_PASSWORD + - name: Publish package + run: mvn -ntp deploy + env: + MAVEN_USERNAME: ${{ secrets.MAVEN_USERNAME }} + MAVEN_PASSWORD: ${{ secrets.MAVEN_PASSWORD }} + if: ${{ contains('SNAPSHOT', needs.version.outputs.version) && github.ref == 'refs/heads/master' && github.repository == 'jruby/jcodings' }} diff --git a/.gitignore b/.gitignore index eb5a316c..824fc6cb 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,4 @@ target +jcodings.iml +.idea/ + diff --git a/.mvn/extensions.xml b/.mvn/extensions.xml deleted file mode 100644 index 57656b9a..00000000 --- a/.mvn/extensions.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - io.takari.polyglot - polyglot-yaml - 0.3.0 - - diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index a0bf2d1c..00000000 --- a/.travis.yml +++ /dev/null @@ -1,7 +0,0 @@ -language: java - -jdk: -- openjdk9 - -script: mvn test - diff --git a/LICENSE.txt b/LICENSE.txt index f8031949..ed2a963f 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,3 +1,5 @@ +Copyright (c) 2025 JRuby Team + Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to diff --git a/README.md b/README.md index 78e53a5f..1b7f325a 100644 --- a/README.md +++ b/README.md @@ -6,3 +6,15 @@ Java-based codings helper classes for Joni and JRuby ## License JCodings is released under the [MIT License](http://www.opensource.org/licenses/MIT). + +## Publishing + +Builds can be published using `mvn deploy`. + +See https://central.sonatype.org/publish/publish-portal-maven/#publishing + +## Releasing + +The version in pom.xml should be updated to remove `-SNAPSHOT` when deploying a release and bumped to the next snapshot version after. + +The `release` profile must be specified with `mvn deploy -Prelease` to include sources, javadocs, and artifact signing required by Maven Central. diff --git a/pom.xml b/pom.xml new file mode 100644 index 00000000..e76e262a --- /dev/null +++ b/pom.xml @@ -0,0 +1,230 @@ + + + 4.0.0 + org.jruby.jcodings + jcodings + 1.0.65-SNAPSHOT + JCodings + Byte based encoding support library for java + https://github.com/jruby/jcodings + + + MIT License + http://www.opensource.org/licenses/mit-license.php + repo + + + + + lopex + Marcin Mielzynski + lopx@gazeta.pl + + + headius + Charles Oliver Nutter + headius@headius.com + + + + scm:git:git://github.com/jruby/jcodings.git + scm:git:git@github.com:jruby/jcodings.git + https://github.com/jruby/jcodings + HEAD + + + Github + https://github.com/jruby/jcodings/issues + + + + junit + junit + 4.13.1 + test + + + + src + scripts + test + + + org.apache.maven.wagon + wagon-webdav-jackrabbit + 2.1 + + + + + false + resources + + + jcodings + + + maven-compiler-plugin + 3.8.1 + + + default-compile + + + module-info.java + + + + + + UTF-8 + 1.8 + 1.8 + + + + maven-jar-plugin + 3.2.0 + + + MANIFEST.MF + + + + + maven-resources-plugin + 3.2.0 + + + maven-surefire-plugin + 3.0.0-M5 + + -Dfile.encoding=UTF-8 + false + + + + org.sonatype.central + central-publishing-maven-plugin + 0.7.0 + true + + central + + + + + + + release-on-9 + + [9,) + + + + + maven-compiler-plugin + + + compile9 + + compile + + + 9 + + module-info.java + + + + + + + maven-source-plugin + 3.2.1 + + + attach-sources + + jar + + + + + + maven-javadoc-plugin + 3.0.1 + + + attach-javadocs + + jar + + + + + -Xdoclint:none + -html5 + true + false + + + + + + + release + + + + maven-source-plugin + + + attach-sources + + jar-no-fork + + + + + + maven-javadoc-plugin + + + attach-javadocs + + jar + + + + + none + + + + maven-gpg-plugin + 3.2.4 + + + sign-artifacts + verify + + sign + + + + + + --pinentry-mode + loopback + + + + + + + true + + + + diff --git a/pom.yml b/pom.yml deleted file mode 100644 index 5f28a6a5..00000000 --- a/pom.yml +++ /dev/null @@ -1,94 +0,0 @@ -modelVersion: 4.0.0 -parent: - artifactId: oss-parent - groupId: org.sonatype.oss - relativePath: ../pom.xml - version: 7 -groupId: org.jruby.jcodings -artifactId: jcodings -version: 1.0.33-SNAPSHOT -packaging: jar -name: JCodings -description: Byte based encoding support library for java -licenses: -- distribution: repo - name: MIT License - url: 'http://www.opensource.org/licenses/mit-license.php' -developers: -- name: Marcin Mielzynski - id: lopex - email: lopx@gazeta.pl -- name: Charles Oliver Nutter - id: headius - email: headius@headius.com -scm: - connection: 'scm:git:git://github.com/jruby/jcodings.git' - developerConnection: 'scm:git:git@github.com:jruby/jcodings.git' - tag: HEAD - url: 'https://github.com/jruby/jcodings' -issueManagement: - system: Github - url: 'https://github.com/jruby/jcodings/issues' -dependencies: -- groupId: junit - artifactId: junit - version: 4.10 - scope: test -build: - extensions: - - artifactId: wagon-webdav-jackrabbit - groupId: org.apache.maven.wagon - version: 2.1 - finalName: jcodings - plugins: - - groupId: org.apache.maven.plugins - artifactId: maven-source-plugin - version: 2.2.1 - executions: - - id: attach-sources - goals: [jar] - - groupId: org.apache.maven.plugins - artifactId: maven-javadoc-plugin - version: 2.9 - configuration: - additionalparam: '-Xdoclint:none' - quiet: true - executions: - - id: attach-javadocs - goals: [jar] - - groupId: org.apache.maven.plugins - artifactId: maven-compiler-plugin - version: 3.8.0 - configuration: - encoding: UTF-8 - executions: - - id: default-compile - configuration: - source: 1.7 - target: 1.7 - excludes: [module-info.java] - - id: compile9 - goals: [compile] - configuration: - release: 9 - includes: [module-info.java] - - groupId: org.apache.maven.plugins - artifactId: maven-jar-plugin - version: 2.4 - configuration: - archive: - manifestFile: MANIFEST.MF - - groupId: org.apache.maven.plugins - artifactId: maven-resources-plugin - version: 2.6 - - groupId: org.apache.maven.plugins - artifactId: maven-surefire-plugin - version: 2.20.1 - configuration: - argLine: -Dfile.encoding=UTF-8 - resources: - - directory: resources - filtering: false - scriptSourceDirectory: scripts - sourceDirectory: src - testSourceDirectory: test diff --git a/resources/tables/CR_Adlam.bin b/resources/tables/CR_Adlam.bin index 41665b96..82bc7f59 100644 Binary files a/resources/tables/CR_Adlam.bin and b/resources/tables/CR_Adlam.bin differ diff --git a/resources/tables/CR_Age_11_0.bin b/resources/tables/CR_Age_11_0.bin new file mode 100644 index 00000000..378be452 Binary files /dev/null and b/resources/tables/CR_Age_11_0.bin differ diff --git a/resources/tables/CR_Age_12_0.bin b/resources/tables/CR_Age_12_0.bin new file mode 100644 index 00000000..3b712519 Binary files /dev/null and b/resources/tables/CR_Age_12_0.bin differ diff --git a/resources/tables/CR_Age_12_1.bin b/resources/tables/CR_Age_12_1.bin new file mode 100644 index 00000000..016d87e7 Binary files /dev/null and b/resources/tables/CR_Age_12_1.bin differ diff --git a/resources/tables/CR_Age_13_0.bin b/resources/tables/CR_Age_13_0.bin new file mode 100644 index 00000000..cd9054ba Binary files /dev/null and b/resources/tables/CR_Age_13_0.bin differ diff --git a/resources/tables/CR_Age_14_0.bin b/resources/tables/CR_Age_14_0.bin new file mode 100644 index 00000000..30faf1fc Binary files /dev/null and b/resources/tables/CR_Age_14_0.bin differ diff --git a/resources/tables/CR_Age_15_0.bin b/resources/tables/CR_Age_15_0.bin new file mode 100644 index 00000000..f3aaf078 Binary files /dev/null and b/resources/tables/CR_Age_15_0.bin differ diff --git a/resources/tables/CR_Age_15_1.bin b/resources/tables/CR_Age_15_1.bin new file mode 100644 index 00000000..493f1ed7 Binary files /dev/null and b/resources/tables/CR_Age_15_1.bin differ diff --git a/resources/tables/CR_Age_16_0.bin b/resources/tables/CR_Age_16_0.bin new file mode 100644 index 00000000..4b150e7e Binary files /dev/null and b/resources/tables/CR_Age_16_0.bin differ diff --git a/resources/tables/CR_Age_17_0.bin b/resources/tables/CR_Age_17_0.bin new file mode 100644 index 00000000..d2299a72 Binary files /dev/null and b/resources/tables/CR_Age_17_0.bin differ diff --git a/resources/tables/CR_Ahom.bin b/resources/tables/CR_Ahom.bin index a7a029cd..9559430f 100644 Binary files a/resources/tables/CR_Ahom.bin and b/resources/tables/CR_Ahom.bin differ diff --git a/resources/tables/CR_Alnum.bin b/resources/tables/CR_Alnum.bin index 87cfc27a..a378a6d4 100644 Binary files a/resources/tables/CR_Alnum.bin and b/resources/tables/CR_Alnum.bin differ diff --git a/resources/tables/CR_Alpha.bin b/resources/tables/CR_Alpha.bin index 0d4d6284..2fd5a36d 100644 Binary files a/resources/tables/CR_Alpha.bin and b/resources/tables/CR_Alpha.bin differ diff --git a/resources/tables/CR_Arabic.bin b/resources/tables/CR_Arabic.bin index 554d2603..74307352 100644 Binary files a/resources/tables/CR_Arabic.bin and b/resources/tables/CR_Arabic.bin differ diff --git a/resources/tables/CR_Armenian.bin b/resources/tables/CR_Armenian.bin index c6bd894d..3e776cfb 100644 Binary files a/resources/tables/CR_Armenian.bin and b/resources/tables/CR_Armenian.bin differ diff --git a/resources/tables/CR_Assigned.bin b/resources/tables/CR_Assigned.bin index 0a2d7034..bb6f7870 100644 Binary files a/resources/tables/CR_Assigned.bin and b/resources/tables/CR_Assigned.bin differ diff --git a/resources/tables/CR_Balinese.bin b/resources/tables/CR_Balinese.bin index b4c8cf43..18811308 100644 Binary files a/resources/tables/CR_Balinese.bin and b/resources/tables/CR_Balinese.bin differ diff --git a/resources/tables/CR_Bengali.bin b/resources/tables/CR_Bengali.bin index f792ed91..da858df8 100644 Binary files a/resources/tables/CR_Bengali.bin and b/resources/tables/CR_Bengali.bin differ diff --git a/resources/tables/CR_Beria_Erfe.bin b/resources/tables/CR_Beria_Erfe.bin new file mode 100644 index 00000000..78106696 Binary files /dev/null and b/resources/tables/CR_Beria_Erfe.bin differ diff --git a/resources/tables/CR_Bopomofo.bin b/resources/tables/CR_Bopomofo.bin index 66aefdeb..2b59dfb0 100644 Binary files a/resources/tables/CR_Bopomofo.bin and b/resources/tables/CR_Bopomofo.bin differ diff --git a/resources/tables/CR_Brahmi.bin b/resources/tables/CR_Brahmi.bin index cedccff5..a9b86176 100644 Binary files a/resources/tables/CR_Brahmi.bin and b/resources/tables/CR_Brahmi.bin differ diff --git a/resources/tables/CR_C.bin b/resources/tables/CR_C.bin index ba890dad..089f7a14 100644 Binary files a/resources/tables/CR_C.bin and b/resources/tables/CR_C.bin differ diff --git a/resources/tables/CR_Canadian_Aboriginal.bin b/resources/tables/CR_Canadian_Aboriginal.bin index 7b9e122e..83bef94e 100644 Binary files a/resources/tables/CR_Canadian_Aboriginal.bin and b/resources/tables/CR_Canadian_Aboriginal.bin differ diff --git a/resources/tables/CR_Case_Ignorable.bin b/resources/tables/CR_Case_Ignorable.bin index 84884f1c..0b12e8fc 100644 Binary files a/resources/tables/CR_Case_Ignorable.bin and b/resources/tables/CR_Case_Ignorable.bin differ diff --git a/resources/tables/CR_Cased.bin b/resources/tables/CR_Cased.bin index 2dca7328..1830b3bc 100644 Binary files a/resources/tables/CR_Cased.bin and b/resources/tables/CR_Cased.bin differ diff --git a/resources/tables/CR_Cf.bin b/resources/tables/CR_Cf.bin index 1dc2929b..d91a2a7a 100644 Binary files a/resources/tables/CR_Cf.bin and b/resources/tables/CR_Cf.bin differ diff --git a/resources/tables/CR_Chakma.bin b/resources/tables/CR_Chakma.bin index bed23eb7..d6d2ae70 100644 Binary files a/resources/tables/CR_Chakma.bin and b/resources/tables/CR_Chakma.bin differ diff --git a/resources/tables/CR_Changes_When_Casefolded.bin b/resources/tables/CR_Changes_When_Casefolded.bin index 3a142797..a172d11e 100644 Binary files a/resources/tables/CR_Changes_When_Casefolded.bin and b/resources/tables/CR_Changes_When_Casefolded.bin differ diff --git a/resources/tables/CR_Changes_When_Casemapped.bin b/resources/tables/CR_Changes_When_Casemapped.bin index cdcf6b65..b4cdefd5 100644 Binary files a/resources/tables/CR_Changes_When_Casemapped.bin and b/resources/tables/CR_Changes_When_Casemapped.bin differ diff --git a/resources/tables/CR_Changes_When_Lowercased.bin b/resources/tables/CR_Changes_When_Lowercased.bin index 9dd3d51b..eb4d936a 100644 Binary files a/resources/tables/CR_Changes_When_Lowercased.bin and b/resources/tables/CR_Changes_When_Lowercased.bin differ diff --git a/resources/tables/CR_Changes_When_Titlecased.bin b/resources/tables/CR_Changes_When_Titlecased.bin index bd0caca0..8202ee00 100644 Binary files a/resources/tables/CR_Changes_When_Titlecased.bin and b/resources/tables/CR_Changes_When_Titlecased.bin differ diff --git a/resources/tables/CR_Changes_When_Uppercased.bin b/resources/tables/CR_Changes_When_Uppercased.bin index d34fd39a..968da222 100644 Binary files a/resources/tables/CR_Changes_When_Uppercased.bin and b/resources/tables/CR_Changes_When_Uppercased.bin differ diff --git a/resources/tables/CR_Chorasmian.bin b/resources/tables/CR_Chorasmian.bin new file mode 100644 index 00000000..08384fe4 Binary files /dev/null and b/resources/tables/CR_Chorasmian.bin differ diff --git a/resources/tables/CR_Cn.bin b/resources/tables/CR_Cn.bin index 720a18ba..d50d1bc7 100644 Binary files a/resources/tables/CR_Cn.bin and b/resources/tables/CR_Cn.bin differ diff --git a/resources/tables/CR_Common.bin b/resources/tables/CR_Common.bin index cee9f022..7c2cb541 100644 Binary files a/resources/tables/CR_Common.bin and b/resources/tables/CR_Common.bin differ diff --git a/resources/tables/CR_Cypro_Minoan.bin b/resources/tables/CR_Cypro_Minoan.bin new file mode 100644 index 00000000..3df632d6 Binary files /dev/null and b/resources/tables/CR_Cypro_Minoan.bin differ diff --git a/resources/tables/CR_Cyrillic.bin b/resources/tables/CR_Cyrillic.bin index b3ace34d..e2e957ae 100644 Binary files a/resources/tables/CR_Cyrillic.bin and b/resources/tables/CR_Cyrillic.bin differ diff --git a/resources/tables/CR_Dash.bin b/resources/tables/CR_Dash.bin index ad5fb6c0..54a68db4 100644 Binary files a/resources/tables/CR_Dash.bin and b/resources/tables/CR_Dash.bin differ diff --git a/resources/tables/CR_Default_Ignorable_Code_Point.bin b/resources/tables/CR_Default_Ignorable_Code_Point.bin index c887540e..7da689e0 100644 Binary files a/resources/tables/CR_Default_Ignorable_Code_Point.bin and b/resources/tables/CR_Default_Ignorable_Code_Point.bin differ diff --git a/resources/tables/CR_Devanagari.bin b/resources/tables/CR_Devanagari.bin index 05c104cc..76412b50 100644 Binary files a/resources/tables/CR_Devanagari.bin and b/resources/tables/CR_Devanagari.bin differ diff --git a/resources/tables/CR_Diacritic.bin b/resources/tables/CR_Diacritic.bin index 2456daab..5f74947e 100644 Binary files a/resources/tables/CR_Diacritic.bin and b/resources/tables/CR_Diacritic.bin differ diff --git a/resources/tables/CR_Digit.bin b/resources/tables/CR_Digit.bin index 62971928..85c73ea5 100644 Binary files a/resources/tables/CR_Digit.bin and b/resources/tables/CR_Digit.bin differ diff --git a/resources/tables/CR_Dives_Akuru.bin b/resources/tables/CR_Dives_Akuru.bin new file mode 100644 index 00000000..f0d026c5 Binary files /dev/null and b/resources/tables/CR_Dives_Akuru.bin differ diff --git a/resources/tables/CR_Dogra.bin b/resources/tables/CR_Dogra.bin new file mode 100644 index 00000000..f252d95f Binary files /dev/null and b/resources/tables/CR_Dogra.bin differ diff --git a/resources/tables/CR_Egyptian_Hieroglyphs.bin b/resources/tables/CR_Egyptian_Hieroglyphs.bin index 653db009..11b0a5df 100644 Binary files a/resources/tables/CR_Egyptian_Hieroglyphs.bin and b/resources/tables/CR_Egyptian_Hieroglyphs.bin differ diff --git a/resources/tables/CR_Elymaic.bin b/resources/tables/CR_Elymaic.bin new file mode 100644 index 00000000..c7195add Binary files /dev/null and b/resources/tables/CR_Elymaic.bin differ diff --git a/resources/tables/CR_Emoji.bin b/resources/tables/CR_Emoji.bin index 276d0f75..514ac355 100644 Binary files a/resources/tables/CR_Emoji.bin and b/resources/tables/CR_Emoji.bin differ diff --git a/resources/tables/CR_Emoji_Component.bin b/resources/tables/CR_Emoji_Component.bin index 142380a4..93a5bc85 100644 Binary files a/resources/tables/CR_Emoji_Component.bin and b/resources/tables/CR_Emoji_Component.bin differ diff --git a/resources/tables/CR_Emoji_Modifier_Base.bin b/resources/tables/CR_Emoji_Modifier_Base.bin index dc79b777..7f934566 100644 Binary files a/resources/tables/CR_Emoji_Modifier_Base.bin and b/resources/tables/CR_Emoji_Modifier_Base.bin differ diff --git a/resources/tables/CR_Emoji_Presentation.bin b/resources/tables/CR_Emoji_Presentation.bin index 3920c9f5..14154aaf 100644 Binary files a/resources/tables/CR_Emoji_Presentation.bin and b/resources/tables/CR_Emoji_Presentation.bin differ diff --git a/resources/tables/CR_Ethiopic.bin b/resources/tables/CR_Ethiopic.bin index bc89b842..7c3620d8 100644 Binary files a/resources/tables/CR_Ethiopic.bin and b/resources/tables/CR_Ethiopic.bin differ diff --git a/resources/tables/CR_Extended_Pictographic.bin b/resources/tables/CR_Extended_Pictographic.bin new file mode 100644 index 00000000..827f4b2d Binary files /dev/null and b/resources/tables/CR_Extended_Pictographic.bin differ diff --git a/resources/tables/CR_Extender.bin b/resources/tables/CR_Extender.bin index 711a725b..0e26e7ca 100644 Binary files a/resources/tables/CR_Extender.bin and b/resources/tables/CR_Extender.bin differ diff --git a/resources/tables/CR_Garay.bin b/resources/tables/CR_Garay.bin new file mode 100644 index 00000000..2b6985af Binary files /dev/null and b/resources/tables/CR_Garay.bin differ diff --git a/resources/tables/CR_Georgian.bin b/resources/tables/CR_Georgian.bin index 4ad142a6..ac108d2e 100644 Binary files a/resources/tables/CR_Georgian.bin and b/resources/tables/CR_Georgian.bin differ diff --git a/resources/tables/CR_Glagolitic.bin b/resources/tables/CR_Glagolitic.bin index 183347a5..a530a0a9 100644 Binary files a/resources/tables/CR_Glagolitic.bin and b/resources/tables/CR_Glagolitic.bin differ diff --git a/resources/tables/CR_Graph.bin b/resources/tables/CR_Graph.bin index d25f97d7..0ccfb29c 100644 Binary files a/resources/tables/CR_Graph.bin and b/resources/tables/CR_Graph.bin differ diff --git a/resources/tables/CR_Grapheme_Base.bin b/resources/tables/CR_Grapheme_Base.bin index ea840a19..75a8e5f3 100644 Binary files a/resources/tables/CR_Grapheme_Base.bin and b/resources/tables/CR_Grapheme_Base.bin differ diff --git a/resources/tables/CR_Grapheme_Cluster_Break_Control.bin b/resources/tables/CR_Grapheme_Cluster_Break_Control.bin index e88a94e4..9ea27aef 100644 Binary files a/resources/tables/CR_Grapheme_Cluster_Break_Control.bin and b/resources/tables/CR_Grapheme_Cluster_Break_Control.bin differ diff --git a/resources/tables/CR_Grapheme_Cluster_Break_E_Base.bin b/resources/tables/CR_Grapheme_Cluster_Break_E_Base.bin deleted file mode 100644 index ce53e851..00000000 Binary files a/resources/tables/CR_Grapheme_Cluster_Break_E_Base.bin and /dev/null differ diff --git a/resources/tables/CR_Grapheme_Cluster_Break_E_Base_GAZ.bin b/resources/tables/CR_Grapheme_Cluster_Break_E_Base_GAZ.bin deleted file mode 100644 index a2813f40..00000000 Binary files a/resources/tables/CR_Grapheme_Cluster_Break_E_Base_GAZ.bin and /dev/null differ diff --git a/resources/tables/CR_Grapheme_Cluster_Break_Extend.bin b/resources/tables/CR_Grapheme_Cluster_Break_Extend.bin new file mode 100644 index 00000000..e225c28e Binary files /dev/null and b/resources/tables/CR_Grapheme_Cluster_Break_Extend.bin differ diff --git a/resources/tables/CR_Grapheme_Cluster_Break_Glue_After_Zwj.bin b/resources/tables/CR_Grapheme_Cluster_Break_Glue_After_Zwj.bin deleted file mode 100644 index 84c2e064..00000000 Binary files a/resources/tables/CR_Grapheme_Cluster_Break_Glue_After_Zwj.bin and /dev/null differ diff --git a/resources/tables/CR_Grapheme_Cluster_Break_Prepend.bin b/resources/tables/CR_Grapheme_Cluster_Break_Prepend.bin index e948e99a..79f2af13 100644 Binary files a/resources/tables/CR_Grapheme_Cluster_Break_Prepend.bin and b/resources/tables/CR_Grapheme_Cluster_Break_Prepend.bin differ diff --git a/resources/tables/CR_Grapheme_Cluster_Break_SpacingMark.bin b/resources/tables/CR_Grapheme_Cluster_Break_SpacingMark.bin index 4132ea8e..d98b581b 100644 Binary files a/resources/tables/CR_Grapheme_Cluster_Break_SpacingMark.bin and b/resources/tables/CR_Grapheme_Cluster_Break_SpacingMark.bin differ diff --git a/resources/tables/CR_Grapheme_Cluster_Break_V.bin b/resources/tables/CR_Grapheme_Cluster_Break_V.bin index dc23a160..a2812dda 100644 Binary files a/resources/tables/CR_Grapheme_Cluster_Break_V.bin and b/resources/tables/CR_Grapheme_Cluster_Break_V.bin differ diff --git a/resources/tables/CR_Grapheme_Extend.bin b/resources/tables/CR_Grapheme_Extend.bin index e76f1c31..32b8c928 100644 Binary files a/resources/tables/CR_Grapheme_Extend.bin and b/resources/tables/CR_Grapheme_Extend.bin differ diff --git a/resources/tables/CR_Grapheme_Link.bin b/resources/tables/CR_Grapheme_Link.bin index 3a8d7dd1..d30b17c7 100644 Binary files a/resources/tables/CR_Grapheme_Link.bin and b/resources/tables/CR_Grapheme_Link.bin differ diff --git a/resources/tables/CR_Gunjala_Gondi.bin b/resources/tables/CR_Gunjala_Gondi.bin new file mode 100644 index 00000000..4255e1cc Binary files /dev/null and b/resources/tables/CR_Gunjala_Gondi.bin differ diff --git a/resources/tables/CR_Gurmukhi.bin b/resources/tables/CR_Gurmukhi.bin index 7787e51f..273a23c0 100644 Binary files a/resources/tables/CR_Gurmukhi.bin and b/resources/tables/CR_Gurmukhi.bin differ diff --git a/resources/tables/CR_Gurung_Khema.bin b/resources/tables/CR_Gurung_Khema.bin new file mode 100644 index 00000000..2d20bf08 Binary files /dev/null and b/resources/tables/CR_Gurung_Khema.bin differ diff --git a/resources/tables/CR_Han.bin b/resources/tables/CR_Han.bin index 5b48d2de..f90d8a93 100644 Binary files a/resources/tables/CR_Han.bin and b/resources/tables/CR_Han.bin differ diff --git a/resources/tables/CR_Hanifi_Rohingya.bin b/resources/tables/CR_Hanifi_Rohingya.bin new file mode 100644 index 00000000..989b1e20 Binary files /dev/null and b/resources/tables/CR_Hanifi_Rohingya.bin differ diff --git a/resources/tables/CR_Hebrew.bin b/resources/tables/CR_Hebrew.bin index 7218c122..028c99cc 100644 Binary files a/resources/tables/CR_Hebrew.bin and b/resources/tables/CR_Hebrew.bin differ diff --git a/resources/tables/CR_Hiragana.bin b/resources/tables/CR_Hiragana.bin index 1cbe9464..31e32627 100644 Binary files a/resources/tables/CR_Hiragana.bin and b/resources/tables/CR_Hiragana.bin differ diff --git a/resources/tables/CR_IDS_Binary_Operator.bin b/resources/tables/CR_IDS_Binary_Operator.bin index 39846e93..b825c7c3 100644 Binary files a/resources/tables/CR_IDS_Binary_Operator.bin and b/resources/tables/CR_IDS_Binary_Operator.bin differ diff --git a/resources/tables/CR_IDS_Unary_Operator.bin b/resources/tables/CR_IDS_Unary_Operator.bin new file mode 100644 index 00000000..ccf146c5 Binary files /dev/null and b/resources/tables/CR_IDS_Unary_Operator.bin differ diff --git a/resources/tables/CR_ID_Compat_Math_Continue.bin b/resources/tables/CR_ID_Compat_Math_Continue.bin new file mode 100644 index 00000000..30f23e5f Binary files /dev/null and b/resources/tables/CR_ID_Compat_Math_Continue.bin differ diff --git a/resources/tables/CR_ID_Compat_Math_Start.bin b/resources/tables/CR_ID_Compat_Math_Start.bin new file mode 100644 index 00000000..2b75b00c Binary files /dev/null and b/resources/tables/CR_ID_Compat_Math_Start.bin differ diff --git a/resources/tables/CR_ID_Continue.bin b/resources/tables/CR_ID_Continue.bin index ff930213..ff57c7f0 100644 Binary files a/resources/tables/CR_ID_Continue.bin and b/resources/tables/CR_ID_Continue.bin differ diff --git a/resources/tables/CR_ID_Start.bin b/resources/tables/CR_ID_Start.bin index 434b8f3c..79105371 100644 Binary files a/resources/tables/CR_ID_Start.bin and b/resources/tables/CR_ID_Start.bin differ diff --git a/resources/tables/CR_Ideographic.bin b/resources/tables/CR_Ideographic.bin index ca6cfcce..0bc9ffe2 100644 Binary files a/resources/tables/CR_Ideographic.bin and b/resources/tables/CR_Ideographic.bin differ diff --git a/resources/tables/CR_InCB_Consonant.bin b/resources/tables/CR_InCB_Consonant.bin new file mode 100644 index 00000000..a2836c9a Binary files /dev/null and b/resources/tables/CR_InCB_Consonant.bin differ diff --git a/resources/tables/CR_InCB_Extend.bin b/resources/tables/CR_InCB_Extend.bin new file mode 100644 index 00000000..c218304a Binary files /dev/null and b/resources/tables/CR_InCB_Extend.bin differ diff --git a/resources/tables/CR_InCB_Linker.bin b/resources/tables/CR_InCB_Linker.bin new file mode 100644 index 00000000..66f6799e Binary files /dev/null and b/resources/tables/CR_InCB_Linker.bin differ diff --git a/resources/tables/CR_In_Ahom.bin b/resources/tables/CR_In_Ahom.bin index 7cf8b2f7..120d1e4f 100644 Binary files a/resources/tables/CR_In_Ahom.bin and b/resources/tables/CR_In_Ahom.bin differ diff --git a/resources/tables/CR_In_Arabic_Extended_B.bin b/resources/tables/CR_In_Arabic_Extended_B.bin new file mode 100644 index 00000000..a059faca Binary files /dev/null and b/resources/tables/CR_In_Arabic_Extended_B.bin differ diff --git a/resources/tables/CR_In_Arabic_Extended_C.bin b/resources/tables/CR_In_Arabic_Extended_C.bin new file mode 100644 index 00000000..01aa3695 Binary files /dev/null and b/resources/tables/CR_In_Arabic_Extended_C.bin differ diff --git a/resources/tables/CR_In_Beria_Erfe.bin b/resources/tables/CR_In_Beria_Erfe.bin new file mode 100644 index 00000000..db1e8bb2 Binary files /dev/null and b/resources/tables/CR_In_Beria_Erfe.bin differ diff --git a/resources/tables/CR_In_CJK_Unified_Ideographs_Extension_G.bin b/resources/tables/CR_In_CJK_Unified_Ideographs_Extension_G.bin new file mode 100644 index 00000000..72a221ef Binary files /dev/null and b/resources/tables/CR_In_CJK_Unified_Ideographs_Extension_G.bin differ diff --git a/resources/tables/CR_In_CJK_Unified_Ideographs_Extension_H.bin b/resources/tables/CR_In_CJK_Unified_Ideographs_Extension_H.bin new file mode 100644 index 00000000..6f55ca49 Binary files /dev/null and b/resources/tables/CR_In_CJK_Unified_Ideographs_Extension_H.bin differ diff --git a/resources/tables/CR_In_CJK_Unified_Ideographs_Extension_I.bin b/resources/tables/CR_In_CJK_Unified_Ideographs_Extension_I.bin new file mode 100644 index 00000000..865d2372 Binary files /dev/null and b/resources/tables/CR_In_CJK_Unified_Ideographs_Extension_I.bin differ diff --git a/resources/tables/CR_In_CJK_Unified_Ideographs_Extension_J.bin b/resources/tables/CR_In_CJK_Unified_Ideographs_Extension_J.bin new file mode 100644 index 00000000..c75ab3ba Binary files /dev/null and b/resources/tables/CR_In_CJK_Unified_Ideographs_Extension_J.bin differ diff --git a/resources/tables/CR_In_Chess_Symbols.bin b/resources/tables/CR_In_Chess_Symbols.bin new file mode 100644 index 00000000..91f78801 Binary files /dev/null and b/resources/tables/CR_In_Chess_Symbols.bin differ diff --git a/resources/tables/CR_In_Chorasmian.bin b/resources/tables/CR_In_Chorasmian.bin new file mode 100644 index 00000000..e00fdd8f Binary files /dev/null and b/resources/tables/CR_In_Chorasmian.bin differ diff --git a/resources/tables/CR_In_Cypro_Minoan.bin b/resources/tables/CR_In_Cypro_Minoan.bin new file mode 100644 index 00000000..214552a1 Binary files /dev/null and b/resources/tables/CR_In_Cypro_Minoan.bin differ diff --git a/resources/tables/CR_In_Cyrillic_Extended_D.bin b/resources/tables/CR_In_Cyrillic_Extended_D.bin new file mode 100644 index 00000000..4df6670d Binary files /dev/null and b/resources/tables/CR_In_Cyrillic_Extended_D.bin differ diff --git a/resources/tables/CR_In_Devanagari_Extended_A.bin b/resources/tables/CR_In_Devanagari_Extended_A.bin new file mode 100644 index 00000000..80c9c3bf Binary files /dev/null and b/resources/tables/CR_In_Devanagari_Extended_A.bin differ diff --git a/resources/tables/CR_In_Dives_Akuru.bin b/resources/tables/CR_In_Dives_Akuru.bin new file mode 100644 index 00000000..2008a47f Binary files /dev/null and b/resources/tables/CR_In_Dives_Akuru.bin differ diff --git a/resources/tables/CR_In_Dogra.bin b/resources/tables/CR_In_Dogra.bin new file mode 100644 index 00000000..54669647 Binary files /dev/null and b/resources/tables/CR_In_Dogra.bin differ diff --git a/resources/tables/CR_In_Egyptian_Hieroglyph_Format_Controls.bin b/resources/tables/CR_In_Egyptian_Hieroglyph_Format_Controls.bin new file mode 100644 index 00000000..e0669894 Binary files /dev/null and b/resources/tables/CR_In_Egyptian_Hieroglyph_Format_Controls.bin differ diff --git a/resources/tables/CR_In_Egyptian_Hieroglyphs_Extended_A.bin b/resources/tables/CR_In_Egyptian_Hieroglyphs_Extended_A.bin new file mode 100644 index 00000000..6efc347a Binary files /dev/null and b/resources/tables/CR_In_Egyptian_Hieroglyphs_Extended_A.bin differ diff --git a/resources/tables/CR_In_Elymaic.bin b/resources/tables/CR_In_Elymaic.bin new file mode 100644 index 00000000..59e39aa8 Binary files /dev/null and b/resources/tables/CR_In_Elymaic.bin differ diff --git a/resources/tables/CR_In_Ethiopic_Extended_B.bin b/resources/tables/CR_In_Ethiopic_Extended_B.bin new file mode 100644 index 00000000..b6325efd Binary files /dev/null and b/resources/tables/CR_In_Ethiopic_Extended_B.bin differ diff --git a/resources/tables/CR_In_Garay.bin b/resources/tables/CR_In_Garay.bin new file mode 100644 index 00000000..d80fbc34 Binary files /dev/null and b/resources/tables/CR_In_Garay.bin differ diff --git a/resources/tables/CR_In_Georgian_Extended.bin b/resources/tables/CR_In_Georgian_Extended.bin new file mode 100644 index 00000000..8c9f7b28 Binary files /dev/null and b/resources/tables/CR_In_Georgian_Extended.bin differ diff --git a/resources/tables/CR_In_Gunjala_Gondi.bin b/resources/tables/CR_In_Gunjala_Gondi.bin new file mode 100644 index 00000000..eda143c5 Binary files /dev/null and b/resources/tables/CR_In_Gunjala_Gondi.bin differ diff --git a/resources/tables/CR_In_Gurung_Khema.bin b/resources/tables/CR_In_Gurung_Khema.bin new file mode 100644 index 00000000..8ddc84b8 Binary files /dev/null and b/resources/tables/CR_In_Gurung_Khema.bin differ diff --git a/resources/tables/CR_In_Hanifi_Rohingya.bin b/resources/tables/CR_In_Hanifi_Rohingya.bin new file mode 100644 index 00000000..1de8818f Binary files /dev/null and b/resources/tables/CR_In_Hanifi_Rohingya.bin differ diff --git a/resources/tables/CR_In_Indic_Siyaq_Numbers.bin b/resources/tables/CR_In_Indic_Siyaq_Numbers.bin new file mode 100644 index 00000000..d527d5ac Binary files /dev/null and b/resources/tables/CR_In_Indic_Siyaq_Numbers.bin differ diff --git a/resources/tables/CR_In_Kaktovik_Numerals.bin b/resources/tables/CR_In_Kaktovik_Numerals.bin new file mode 100644 index 00000000..9beb9711 Binary files /dev/null and b/resources/tables/CR_In_Kaktovik_Numerals.bin differ diff --git a/resources/tables/CR_In_Kana_Extended_B.bin b/resources/tables/CR_In_Kana_Extended_B.bin new file mode 100644 index 00000000..2ea6aff6 Binary files /dev/null and b/resources/tables/CR_In_Kana_Extended_B.bin differ diff --git a/resources/tables/CR_In_Kawi.bin b/resources/tables/CR_In_Kawi.bin new file mode 100644 index 00000000..58df9471 Binary files /dev/null and b/resources/tables/CR_In_Kawi.bin differ diff --git a/resources/tables/CR_In_Khitan_Small_Script.bin b/resources/tables/CR_In_Khitan_Small_Script.bin new file mode 100644 index 00000000..4fb7d41e Binary files /dev/null and b/resources/tables/CR_In_Khitan_Small_Script.bin differ diff --git a/resources/tables/CR_In_Kirat_Rai.bin b/resources/tables/CR_In_Kirat_Rai.bin new file mode 100644 index 00000000..7cf87f7d Binary files /dev/null and b/resources/tables/CR_In_Kirat_Rai.bin differ diff --git a/resources/tables/CR_In_Latin_Extended_F.bin b/resources/tables/CR_In_Latin_Extended_F.bin new file mode 100644 index 00000000..bf7ec02d Binary files /dev/null and b/resources/tables/CR_In_Latin_Extended_F.bin differ diff --git a/resources/tables/CR_In_Latin_Extended_G.bin b/resources/tables/CR_In_Latin_Extended_G.bin new file mode 100644 index 00000000..c9491171 Binary files /dev/null and b/resources/tables/CR_In_Latin_Extended_G.bin differ diff --git a/resources/tables/CR_In_Lisu.bin b/resources/tables/CR_In_Lisu.bin new file mode 100644 index 00000000..5ba17331 Binary files /dev/null and b/resources/tables/CR_In_Lisu.bin differ diff --git a/resources/tables/CR_In_Lisu_Supplement.bin b/resources/tables/CR_In_Lisu_Supplement.bin new file mode 100644 index 00000000..f584d223 Binary files /dev/null and b/resources/tables/CR_In_Lisu_Supplement.bin differ diff --git a/resources/tables/CR_In_Makasar.bin b/resources/tables/CR_In_Makasar.bin new file mode 100644 index 00000000..11113a4b Binary files /dev/null and b/resources/tables/CR_In_Makasar.bin differ diff --git a/resources/tables/CR_In_Mayan_Numerals.bin b/resources/tables/CR_In_Mayan_Numerals.bin new file mode 100644 index 00000000..d99e51ac Binary files /dev/null and b/resources/tables/CR_In_Mayan_Numerals.bin differ diff --git a/resources/tables/CR_In_Medefaidrin.bin b/resources/tables/CR_In_Medefaidrin.bin new file mode 100644 index 00000000..5aef9f10 Binary files /dev/null and b/resources/tables/CR_In_Medefaidrin.bin differ diff --git a/resources/tables/CR_In_Miscellaneous_Symbols_Supplement.bin b/resources/tables/CR_In_Miscellaneous_Symbols_Supplement.bin new file mode 100644 index 00000000..97f6a646 Binary files /dev/null and b/resources/tables/CR_In_Miscellaneous_Symbols_Supplement.bin differ diff --git a/resources/tables/CR_In_Miscellaneous_Symbols_And_Pictographs.bin b/resources/tables/CR_In_Miscellaneous_Symbols_and_Pictographs.bin similarity index 100% rename from resources/tables/CR_In_Miscellaneous_Symbols_And_Pictographs.bin rename to resources/tables/CR_In_Miscellaneous_Symbols_and_Pictographs.bin diff --git a/resources/tables/CR_In_Myanmar_Extended_C.bin b/resources/tables/CR_In_Myanmar_Extended_C.bin new file mode 100644 index 00000000..0585a6a1 Binary files /dev/null and b/resources/tables/CR_In_Myanmar_Extended_C.bin differ diff --git a/resources/tables/CR_In_Nag_Mundari.bin b/resources/tables/CR_In_Nag_Mundari.bin new file mode 100644 index 00000000..52b03112 Binary files /dev/null and b/resources/tables/CR_In_Nag_Mundari.bin differ diff --git a/resources/tables/CR_In_Nandinagari.bin b/resources/tables/CR_In_Nandinagari.bin new file mode 100644 index 00000000..d23448c9 Binary files /dev/null and b/resources/tables/CR_In_Nandinagari.bin differ diff --git a/resources/tables/CR_In_No_Block.bin b/resources/tables/CR_In_No_Block.bin index 186f4e22..2befca5b 100644 Binary files a/resources/tables/CR_In_No_Block.bin and b/resources/tables/CR_In_No_Block.bin differ diff --git a/resources/tables/CR_In_Nyiakeng_Puachue_Hmong.bin b/resources/tables/CR_In_Nyiakeng_Puachue_Hmong.bin new file mode 100644 index 00000000..e6ef8e32 Binary files /dev/null and b/resources/tables/CR_In_Nyiakeng_Puachue_Hmong.bin differ diff --git a/resources/tables/CR_In_Ol_Onal.bin b/resources/tables/CR_In_Ol_Onal.bin new file mode 100644 index 00000000..c38b4cde Binary files /dev/null and b/resources/tables/CR_In_Ol_Onal.bin differ diff --git a/resources/tables/CR_In_Old_Sogdian.bin b/resources/tables/CR_In_Old_Sogdian.bin new file mode 100644 index 00000000..afba048e Binary files /dev/null and b/resources/tables/CR_In_Old_Sogdian.bin differ diff --git a/resources/tables/CR_In_Old_Uyghur.bin b/resources/tables/CR_In_Old_Uyghur.bin new file mode 100644 index 00000000..ec970fb3 Binary files /dev/null and b/resources/tables/CR_In_Old_Uyghur.bin differ diff --git a/resources/tables/CR_In_Ottoman_Siyaq_Numbers.bin b/resources/tables/CR_In_Ottoman_Siyaq_Numbers.bin new file mode 100644 index 00000000..8c7f19da Binary files /dev/null and b/resources/tables/CR_In_Ottoman_Siyaq_Numbers.bin differ diff --git a/resources/tables/CR_In_Sharada_Supplement.bin b/resources/tables/CR_In_Sharada_Supplement.bin new file mode 100644 index 00000000..c3217b02 Binary files /dev/null and b/resources/tables/CR_In_Sharada_Supplement.bin differ diff --git a/resources/tables/CR_In_Sidetic.bin b/resources/tables/CR_In_Sidetic.bin new file mode 100644 index 00000000..4b7e9f55 Binary files /dev/null and b/resources/tables/CR_In_Sidetic.bin differ diff --git a/resources/tables/CR_In_Small_Kana_Extension.bin b/resources/tables/CR_In_Small_Kana_Extension.bin new file mode 100644 index 00000000..72e7421f Binary files /dev/null and b/resources/tables/CR_In_Small_Kana_Extension.bin differ diff --git a/resources/tables/CR_In_Sogdian.bin b/resources/tables/CR_In_Sogdian.bin new file mode 100644 index 00000000..a6c1ebed Binary files /dev/null and b/resources/tables/CR_In_Sogdian.bin differ diff --git a/resources/tables/CR_In_Sunuwar.bin b/resources/tables/CR_In_Sunuwar.bin new file mode 100644 index 00000000..488bd70b Binary files /dev/null and b/resources/tables/CR_In_Sunuwar.bin differ diff --git a/resources/tables/CR_In_Symbols_and_Pictographs_Extended_A.bin b/resources/tables/CR_In_Symbols_and_Pictographs_Extended_A.bin new file mode 100644 index 00000000..7b1484aa Binary files /dev/null and b/resources/tables/CR_In_Symbols_and_Pictographs_Extended_A.bin differ diff --git a/resources/tables/CR_In_Symbols_for_Legacy_Computing.bin b/resources/tables/CR_In_Symbols_for_Legacy_Computing.bin new file mode 100644 index 00000000..81e19d23 Binary files /dev/null and b/resources/tables/CR_In_Symbols_for_Legacy_Computing.bin differ diff --git a/resources/tables/CR_In_Symbols_for_Legacy_Computing_Supplement.bin b/resources/tables/CR_In_Symbols_for_Legacy_Computing_Supplement.bin new file mode 100644 index 00000000..026da3da Binary files /dev/null and b/resources/tables/CR_In_Symbols_for_Legacy_Computing_Supplement.bin differ diff --git a/resources/tables/CR_In_Tai_Yo.bin b/resources/tables/CR_In_Tai_Yo.bin new file mode 100644 index 00000000..56a83384 Binary files /dev/null and b/resources/tables/CR_In_Tai_Yo.bin differ diff --git a/resources/tables/CR_In_Tamil_Supplement.bin b/resources/tables/CR_In_Tamil_Supplement.bin new file mode 100644 index 00000000..eebafe44 Binary files /dev/null and b/resources/tables/CR_In_Tamil_Supplement.bin differ diff --git a/resources/tables/CR_In_Tangsa.bin b/resources/tables/CR_In_Tangsa.bin new file mode 100644 index 00000000..aa5a0857 Binary files /dev/null and b/resources/tables/CR_In_Tangsa.bin differ diff --git a/resources/tables/CR_In_Tangut_Components_Supplement.bin b/resources/tables/CR_In_Tangut_Components_Supplement.bin new file mode 100644 index 00000000..f62d2748 Binary files /dev/null and b/resources/tables/CR_In_Tangut_Components_Supplement.bin differ diff --git a/resources/tables/CR_In_Tangut_Supplement.bin b/resources/tables/CR_In_Tangut_Supplement.bin new file mode 100644 index 00000000..74e40bbc Binary files /dev/null and b/resources/tables/CR_In_Tangut_Supplement.bin differ diff --git a/resources/tables/CR_In_Todhri.bin b/resources/tables/CR_In_Todhri.bin new file mode 100644 index 00000000..b3f00ebe Binary files /dev/null and b/resources/tables/CR_In_Todhri.bin differ diff --git a/resources/tables/CR_In_Tolong_Siki.bin b/resources/tables/CR_In_Tolong_Siki.bin new file mode 100644 index 00000000..763d1660 Binary files /dev/null and b/resources/tables/CR_In_Tolong_Siki.bin differ diff --git a/resources/tables/CR_In_Toto.bin b/resources/tables/CR_In_Toto.bin new file mode 100644 index 00000000..c81bbe73 Binary files /dev/null and b/resources/tables/CR_In_Toto.bin differ diff --git a/resources/tables/CR_In_Transport_And_Map_Symbols.bin b/resources/tables/CR_In_Transport_and_Map_Symbols.bin similarity index 100% rename from resources/tables/CR_In_Transport_And_Map_Symbols.bin rename to resources/tables/CR_In_Transport_and_Map_Symbols.bin diff --git a/resources/tables/CR_In_Tulu_Tigalari.bin b/resources/tables/CR_In_Tulu_Tigalari.bin new file mode 100644 index 00000000..572462ec Binary files /dev/null and b/resources/tables/CR_In_Tulu_Tigalari.bin differ diff --git a/resources/tables/CR_In_Unified_Canadian_Aboriginal_Syllabics_Extended_A.bin b/resources/tables/CR_In_Unified_Canadian_Aboriginal_Syllabics_Extended_A.bin new file mode 100644 index 00000000..64428eb0 Binary files /dev/null and b/resources/tables/CR_In_Unified_Canadian_Aboriginal_Syllabics_Extended_A.bin differ diff --git a/resources/tables/CR_In_Vithkuqi.bin b/resources/tables/CR_In_Vithkuqi.bin new file mode 100644 index 00000000..d833c467 Binary files /dev/null and b/resources/tables/CR_In_Vithkuqi.bin differ diff --git a/resources/tables/CR_In_Wancho.bin b/resources/tables/CR_In_Wancho.bin new file mode 100644 index 00000000..59b7d43b Binary files /dev/null and b/resources/tables/CR_In_Wancho.bin differ diff --git a/resources/tables/CR_In_Yezidi.bin b/resources/tables/CR_In_Yezidi.bin new file mode 100644 index 00000000..6ee958c5 Binary files /dev/null and b/resources/tables/CR_In_Yezidi.bin differ diff --git a/resources/tables/CR_In_Znamenny_Musical_Notation.bin b/resources/tables/CR_In_Znamenny_Musical_Notation.bin new file mode 100644 index 00000000..079a0db4 Binary files /dev/null and b/resources/tables/CR_In_Znamenny_Musical_Notation.bin differ diff --git a/resources/tables/CR_Inherited.bin b/resources/tables/CR_Inherited.bin index 07dd6ada..1b784849 100644 Binary files a/resources/tables/CR_Inherited.bin and b/resources/tables/CR_Inherited.bin differ diff --git a/resources/tables/CR_Kaithi.bin b/resources/tables/CR_Kaithi.bin index 0c2569a5..e10a54c6 100644 Binary files a/resources/tables/CR_Kaithi.bin and b/resources/tables/CR_Kaithi.bin differ diff --git a/resources/tables/CR_Kannada.bin b/resources/tables/CR_Kannada.bin index ed5ebebc..b1326d7d 100644 Binary files a/resources/tables/CR_Kannada.bin and b/resources/tables/CR_Kannada.bin differ diff --git a/resources/tables/CR_Katakana.bin b/resources/tables/CR_Katakana.bin index 024d968d..858d3e00 100644 Binary files a/resources/tables/CR_Katakana.bin and b/resources/tables/CR_Katakana.bin differ diff --git a/resources/tables/CR_Kawi.bin b/resources/tables/CR_Kawi.bin new file mode 100644 index 00000000..cefc48f0 Binary files /dev/null and b/resources/tables/CR_Kawi.bin differ diff --git a/resources/tables/CR_Kharoshthi.bin b/resources/tables/CR_Kharoshthi.bin index 0ed91f6b..2dc09929 100644 Binary files a/resources/tables/CR_Kharoshthi.bin and b/resources/tables/CR_Kharoshthi.bin differ diff --git a/resources/tables/CR_Khitan_Small_Script.bin b/resources/tables/CR_Khitan_Small_Script.bin new file mode 100644 index 00000000..b35b9271 Binary files /dev/null and b/resources/tables/CR_Khitan_Small_Script.bin differ diff --git a/resources/tables/CR_Khojki.bin b/resources/tables/CR_Khojki.bin index 2dd1190f..880bcbe3 100644 Binary files a/resources/tables/CR_Khojki.bin and b/resources/tables/CR_Khojki.bin differ diff --git a/resources/tables/CR_Kirat_Rai.bin b/resources/tables/CR_Kirat_Rai.bin new file mode 100644 index 00000000..da70d0fd Binary files /dev/null and b/resources/tables/CR_Kirat_Rai.bin differ diff --git a/resources/tables/CR_L.bin b/resources/tables/CR_L.bin index 7bcf4bbd..667ee05a 100644 Binary files a/resources/tables/CR_L.bin and b/resources/tables/CR_L.bin differ diff --git a/resources/tables/CR_LC.bin b/resources/tables/CR_LC.bin index c5cac563..c139e40f 100644 Binary files a/resources/tables/CR_LC.bin and b/resources/tables/CR_LC.bin differ diff --git a/resources/tables/CR_Lao.bin b/resources/tables/CR_Lao.bin index db9468e2..a6d64e6c 100644 Binary files a/resources/tables/CR_Lao.bin and b/resources/tables/CR_Lao.bin differ diff --git a/resources/tables/CR_Latin.bin b/resources/tables/CR_Latin.bin index 8ce6495d..49871f34 100644 Binary files a/resources/tables/CR_Latin.bin and b/resources/tables/CR_Latin.bin differ diff --git a/resources/tables/CR_Lisu.bin b/resources/tables/CR_Lisu.bin index 5ba17331..df8cddee 100644 Binary files a/resources/tables/CR_Lisu.bin and b/resources/tables/CR_Lisu.bin differ diff --git a/resources/tables/CR_Ll.bin b/resources/tables/CR_Ll.bin index 469e199c..b9cf5fa2 100644 Binary files a/resources/tables/CR_Ll.bin and b/resources/tables/CR_Ll.bin differ diff --git a/resources/tables/CR_Lm.bin b/resources/tables/CR_Lm.bin index 28c71890..9673765f 100644 Binary files a/resources/tables/CR_Lm.bin and b/resources/tables/CR_Lm.bin differ diff --git a/resources/tables/CR_Lo.bin b/resources/tables/CR_Lo.bin index 729d653c..402df6b7 100644 Binary files a/resources/tables/CR_Lo.bin and b/resources/tables/CR_Lo.bin differ diff --git a/resources/tables/CR_Lower.bin b/resources/tables/CR_Lower.bin index b21c4def..857593e9 100644 Binary files a/resources/tables/CR_Lower.bin and b/resources/tables/CR_Lower.bin differ diff --git a/resources/tables/CR_Lu.bin b/resources/tables/CR_Lu.bin index 209614d2..26ca7e3f 100644 Binary files a/resources/tables/CR_Lu.bin and b/resources/tables/CR_Lu.bin differ diff --git a/resources/tables/CR_M.bin b/resources/tables/CR_M.bin index 69e44a39..4f3ff89b 100644 Binary files a/resources/tables/CR_M.bin and b/resources/tables/CR_M.bin differ diff --git a/resources/tables/CR_Makasar.bin b/resources/tables/CR_Makasar.bin new file mode 100644 index 00000000..2fb02b1d Binary files /dev/null and b/resources/tables/CR_Makasar.bin differ diff --git a/resources/tables/CR_Malayalam.bin b/resources/tables/CR_Malayalam.bin index fbfe97fb..f16069ba 100644 Binary files a/resources/tables/CR_Malayalam.bin and b/resources/tables/CR_Malayalam.bin differ diff --git a/resources/tables/CR_Math.bin b/resources/tables/CR_Math.bin index 06a31e1a..d6e2fb95 100644 Binary files a/resources/tables/CR_Math.bin and b/resources/tables/CR_Math.bin differ diff --git a/resources/tables/CR_Mc.bin b/resources/tables/CR_Mc.bin index 713c3ce3..6a4a1481 100644 Binary files a/resources/tables/CR_Mc.bin and b/resources/tables/CR_Mc.bin differ diff --git a/resources/tables/CR_Medefaidrin.bin b/resources/tables/CR_Medefaidrin.bin new file mode 100644 index 00000000..22e3982e Binary files /dev/null and b/resources/tables/CR_Medefaidrin.bin differ diff --git a/resources/tables/CR_Miao.bin b/resources/tables/CR_Miao.bin index 1444a883..d606ae32 100644 Binary files a/resources/tables/CR_Miao.bin and b/resources/tables/CR_Miao.bin differ diff --git a/resources/tables/CR_Mn.bin b/resources/tables/CR_Mn.bin index 571425af..3be25833 100644 Binary files a/resources/tables/CR_Mn.bin and b/resources/tables/CR_Mn.bin differ diff --git a/resources/tables/CR_Modifier_Combining_Mark.bin b/resources/tables/CR_Modifier_Combining_Mark.bin new file mode 100644 index 00000000..96d46d43 Binary files /dev/null and b/resources/tables/CR_Modifier_Combining_Mark.bin differ diff --git a/resources/tables/CR_Mongolian.bin b/resources/tables/CR_Mongolian.bin index 9e10184f..6d8886c4 100644 Binary files a/resources/tables/CR_Mongolian.bin and b/resources/tables/CR_Mongolian.bin differ diff --git a/resources/tables/CR_Myanmar.bin b/resources/tables/CR_Myanmar.bin index 6f6a2559..3dd0998f 100644 Binary files a/resources/tables/CR_Myanmar.bin and b/resources/tables/CR_Myanmar.bin differ diff --git a/resources/tables/CR_N.bin b/resources/tables/CR_N.bin index 84448d70..048e6137 100644 Binary files a/resources/tables/CR_N.bin and b/resources/tables/CR_N.bin differ diff --git a/resources/tables/CR_Nag_Mundari.bin b/resources/tables/CR_Nag_Mundari.bin new file mode 100644 index 00000000..ab468049 Binary files /dev/null and b/resources/tables/CR_Nag_Mundari.bin differ diff --git a/resources/tables/CR_Nandinagari.bin b/resources/tables/CR_Nandinagari.bin new file mode 100644 index 00000000..9856155b Binary files /dev/null and b/resources/tables/CR_Nandinagari.bin differ diff --git a/resources/tables/CR_Newa.bin b/resources/tables/CR_Newa.bin index e8e534eb..e725ee0e 100644 Binary files a/resources/tables/CR_Newa.bin and b/resources/tables/CR_Newa.bin differ diff --git a/resources/tables/CR_Nko.bin b/resources/tables/CR_Nko.bin index 6afc5072..38c35e10 100644 Binary files a/resources/tables/CR_Nko.bin and b/resources/tables/CR_Nko.bin differ diff --git a/resources/tables/CR_Nl.bin b/resources/tables/CR_Nl.bin index b0feb284..5ac08989 100644 Binary files a/resources/tables/CR_Nl.bin and b/resources/tables/CR_Nl.bin differ diff --git a/resources/tables/CR_No.bin b/resources/tables/CR_No.bin index a62aaa1f..426e624a 100644 Binary files a/resources/tables/CR_No.bin and b/resources/tables/CR_No.bin differ diff --git a/resources/tables/CR_Nyiakeng_Puachue_Hmong.bin b/resources/tables/CR_Nyiakeng_Puachue_Hmong.bin new file mode 100644 index 00000000..936b0f12 Binary files /dev/null and b/resources/tables/CR_Nyiakeng_Puachue_Hmong.bin differ diff --git a/resources/tables/CR_Ol_Onal.bin b/resources/tables/CR_Ol_Onal.bin new file mode 100644 index 00000000..0a6eedb2 Binary files /dev/null and b/resources/tables/CR_Ol_Onal.bin differ diff --git a/resources/tables/CR_Old_Sogdian.bin b/resources/tables/CR_Old_Sogdian.bin new file mode 100644 index 00000000..9f1df984 Binary files /dev/null and b/resources/tables/CR_Old_Sogdian.bin differ diff --git a/resources/tables/CR_Old_Uyghur.bin b/resources/tables/CR_Old_Uyghur.bin new file mode 100644 index 00000000..58562966 Binary files /dev/null and b/resources/tables/CR_Old_Uyghur.bin differ diff --git a/resources/tables/CR_Oriya.bin b/resources/tables/CR_Oriya.bin index 12000009..729ef7bd 100644 Binary files a/resources/tables/CR_Oriya.bin and b/resources/tables/CR_Oriya.bin differ diff --git a/resources/tables/CR_Other_Alphabetic.bin b/resources/tables/CR_Other_Alphabetic.bin index 64749cc3..850174bc 100644 Binary files a/resources/tables/CR_Other_Alphabetic.bin and b/resources/tables/CR_Other_Alphabetic.bin differ diff --git a/resources/tables/CR_Other_Grapheme_Extend.bin b/resources/tables/CR_Other_Grapheme_Extend.bin index 0b0a288a..872ba552 100644 Binary files a/resources/tables/CR_Other_Grapheme_Extend.bin and b/resources/tables/CR_Other_Grapheme_Extend.bin differ diff --git a/resources/tables/CR_Other_ID_Continue.bin b/resources/tables/CR_Other_ID_Continue.bin index 7c77add9..12aa7eb3 100644 Binary files a/resources/tables/CR_Other_ID_Continue.bin and b/resources/tables/CR_Other_ID_Continue.bin differ diff --git a/resources/tables/CR_Other_Lowercase.bin b/resources/tables/CR_Other_Lowercase.bin index be9ff4ab..d15d9e8f 100644 Binary files a/resources/tables/CR_Other_Lowercase.bin and b/resources/tables/CR_Other_Lowercase.bin differ diff --git a/resources/tables/CR_Pd.bin b/resources/tables/CR_Pd.bin index 24d3b70d..3b241d5a 100644 Binary files a/resources/tables/CR_Pd.bin and b/resources/tables/CR_Pd.bin differ diff --git a/resources/tables/CR_Pe.bin b/resources/tables/CR_Pe.bin index 42920a4a..13112651 100644 Binary files a/resources/tables/CR_Pe.bin and b/resources/tables/CR_Pe.bin differ diff --git a/resources/tables/CR_Po.bin b/resources/tables/CR_Po.bin index 15b30457..7643a309 100644 Binary files a/resources/tables/CR_Po.bin and b/resources/tables/CR_Po.bin differ diff --git a/resources/tables/CR_Prepended_Concatenation_Mark.bin b/resources/tables/CR_Prepended_Concatenation_Mark.bin index f9de2023..a90ccf1d 100644 Binary files a/resources/tables/CR_Prepended_Concatenation_Mark.bin and b/resources/tables/CR_Prepended_Concatenation_Mark.bin differ diff --git a/resources/tables/CR_Print.bin b/resources/tables/CR_Print.bin index 0580a332..354a3389 100644 Binary files a/resources/tables/CR_Print.bin and b/resources/tables/CR_Print.bin differ diff --git a/resources/tables/CR_Ps.bin b/resources/tables/CR_Ps.bin index 9f514482..7998abc3 100644 Binary files a/resources/tables/CR_Ps.bin and b/resources/tables/CR_Ps.bin differ diff --git a/resources/tables/CR_Punct.bin b/resources/tables/CR_Punct.bin index 3c0a30f9..21990c25 100644 Binary files a/resources/tables/CR_Punct.bin and b/resources/tables/CR_Punct.bin differ diff --git a/resources/tables/CR_S.bin b/resources/tables/CR_S.bin index 377ba362..8e9a5edd 100644 Binary files a/resources/tables/CR_S.bin and b/resources/tables/CR_S.bin differ diff --git a/resources/tables/CR_Sc.bin b/resources/tables/CR_Sc.bin index e10e76a4..bfd9fecb 100644 Binary files a/resources/tables/CR_Sc.bin and b/resources/tables/CR_Sc.bin differ diff --git a/resources/tables/CR_Sentence_Terminal.bin b/resources/tables/CR_Sentence_Terminal.bin index 22be5298..b4549583 100644 Binary files a/resources/tables/CR_Sentence_Terminal.bin and b/resources/tables/CR_Sentence_Terminal.bin differ diff --git a/resources/tables/CR_Sharada.bin b/resources/tables/CR_Sharada.bin index f5d49955..fb9f67a7 100644 Binary files a/resources/tables/CR_Sharada.bin and b/resources/tables/CR_Sharada.bin differ diff --git a/resources/tables/CR_Sidetic.bin b/resources/tables/CR_Sidetic.bin new file mode 100644 index 00000000..ccafc67a Binary files /dev/null and b/resources/tables/CR_Sidetic.bin differ diff --git a/resources/tables/CR_Sinhala.bin b/resources/tables/CR_Sinhala.bin index e127f64c..211790e2 100644 Binary files a/resources/tables/CR_Sinhala.bin and b/resources/tables/CR_Sinhala.bin differ diff --git a/resources/tables/CR_Sk.bin b/resources/tables/CR_Sk.bin index 7b6d134b..a5963562 100644 Binary files a/resources/tables/CR_Sk.bin and b/resources/tables/CR_Sk.bin differ diff --git a/resources/tables/CR_Sm.bin b/resources/tables/CR_Sm.bin index 5966416a..fe2af3ac 100644 Binary files a/resources/tables/CR_Sm.bin and b/resources/tables/CR_Sm.bin differ diff --git a/resources/tables/CR_So.bin b/resources/tables/CR_So.bin index a93ed019..ccc0781a 100644 Binary files a/resources/tables/CR_So.bin and b/resources/tables/CR_So.bin differ diff --git a/resources/tables/CR_Soft_Dotted.bin b/resources/tables/CR_Soft_Dotted.bin index 703d83af..a4308cb0 100644 Binary files a/resources/tables/CR_Soft_Dotted.bin and b/resources/tables/CR_Soft_Dotted.bin differ diff --git a/resources/tables/CR_Sogdian.bin b/resources/tables/CR_Sogdian.bin new file mode 100644 index 00000000..1244a2f1 Binary files /dev/null and b/resources/tables/CR_Sogdian.bin differ diff --git a/resources/tables/CR_Soyombo.bin b/resources/tables/CR_Soyombo.bin index 74e54e46..ad427968 100644 Binary files a/resources/tables/CR_Soyombo.bin and b/resources/tables/CR_Soyombo.bin differ diff --git a/resources/tables/CR_Sunuwar.bin b/resources/tables/CR_Sunuwar.bin new file mode 100644 index 00000000..b4f8779a Binary files /dev/null and b/resources/tables/CR_Sunuwar.bin differ diff --git a/resources/tables/CR_Syloti_Nagri.bin b/resources/tables/CR_Syloti_Nagri.bin index b31d1b54..99816994 100644 Binary files a/resources/tables/CR_Syloti_Nagri.bin and b/resources/tables/CR_Syloti_Nagri.bin differ diff --git a/resources/tables/CR_Tagalog.bin b/resources/tables/CR_Tagalog.bin index f2b056d2..b0036eda 100644 Binary files a/resources/tables/CR_Tagalog.bin and b/resources/tables/CR_Tagalog.bin differ diff --git a/resources/tables/CR_Tai_Yo.bin b/resources/tables/CR_Tai_Yo.bin new file mode 100644 index 00000000..edd0182d Binary files /dev/null and b/resources/tables/CR_Tai_Yo.bin differ diff --git a/resources/tables/CR_Takri.bin b/resources/tables/CR_Takri.bin index 4e895477..24f19ad6 100644 Binary files a/resources/tables/CR_Takri.bin and b/resources/tables/CR_Takri.bin differ diff --git a/resources/tables/CR_Tamil.bin b/resources/tables/CR_Tamil.bin index 92ffc930..ecdb981c 100644 Binary files a/resources/tables/CR_Tamil.bin and b/resources/tables/CR_Tamil.bin differ diff --git a/resources/tables/CR_Tangsa.bin b/resources/tables/CR_Tangsa.bin new file mode 100644 index 00000000..4e6fea3b Binary files /dev/null and b/resources/tables/CR_Tangsa.bin differ diff --git a/resources/tables/CR_Tangut.bin b/resources/tables/CR_Tangut.bin index e9c7d5cd..e99cf2a6 100644 Binary files a/resources/tables/CR_Tangut.bin and b/resources/tables/CR_Tangut.bin differ diff --git a/resources/tables/CR_Telugu.bin b/resources/tables/CR_Telugu.bin index e1aa0da3..756474ab 100644 Binary files a/resources/tables/CR_Telugu.bin and b/resources/tables/CR_Telugu.bin differ diff --git a/resources/tables/CR_Terminal_Punctuation.bin b/resources/tables/CR_Terminal_Punctuation.bin index 81835aa3..46da787c 100644 Binary files a/resources/tables/CR_Terminal_Punctuation.bin and b/resources/tables/CR_Terminal_Punctuation.bin differ diff --git a/resources/tables/CR_Todhri.bin b/resources/tables/CR_Todhri.bin new file mode 100644 index 00000000..bf10c818 Binary files /dev/null and b/resources/tables/CR_Todhri.bin differ diff --git a/resources/tables/CR_Tolong_Siki.bin b/resources/tables/CR_Tolong_Siki.bin new file mode 100644 index 00000000..34de63b6 Binary files /dev/null and b/resources/tables/CR_Tolong_Siki.bin differ diff --git a/resources/tables/CR_Toto.bin b/resources/tables/CR_Toto.bin new file mode 100644 index 00000000..835bffc0 Binary files /dev/null and b/resources/tables/CR_Toto.bin differ diff --git a/resources/tables/CR_Tulu_Tigalari.bin b/resources/tables/CR_Tulu_Tigalari.bin new file mode 100644 index 00000000..6508b73e Binary files /dev/null and b/resources/tables/CR_Tulu_Tigalari.bin differ diff --git a/resources/tables/CR_Unified_Ideograph.bin b/resources/tables/CR_Unified_Ideograph.bin index b31e7129..06f037de 100644 Binary files a/resources/tables/CR_Unified_Ideograph.bin and b/resources/tables/CR_Unified_Ideograph.bin differ diff --git a/resources/tables/CR_Unknown.bin b/resources/tables/CR_Unknown.bin index daa80e58..2989d7b0 100644 Binary files a/resources/tables/CR_Unknown.bin and b/resources/tables/CR_Unknown.bin differ diff --git a/resources/tables/CR_Upper.bin b/resources/tables/CR_Upper.bin index 5a9637e3..c53af25f 100644 Binary files a/resources/tables/CR_Upper.bin and b/resources/tables/CR_Upper.bin differ diff --git a/resources/tables/CR_Variation_Selector.bin b/resources/tables/CR_Variation_Selector.bin index d7ffc577..2425c77b 100644 Binary files a/resources/tables/CR_Variation_Selector.bin and b/resources/tables/CR_Variation_Selector.bin differ diff --git a/resources/tables/CR_Vithkuqi.bin b/resources/tables/CR_Vithkuqi.bin new file mode 100644 index 00000000..1fb26db9 Binary files /dev/null and b/resources/tables/CR_Vithkuqi.bin differ diff --git a/resources/tables/CR_Wancho.bin b/resources/tables/CR_Wancho.bin new file mode 100644 index 00000000..182d622b Binary files /dev/null and b/resources/tables/CR_Wancho.bin differ diff --git a/resources/tables/CR_Word.bin b/resources/tables/CR_Word.bin index 8274bfb4..f7c8f6d8 100644 Binary files a/resources/tables/CR_Word.bin and b/resources/tables/CR_Word.bin differ diff --git a/resources/tables/CR_XID_Continue.bin b/resources/tables/CR_XID_Continue.bin index 964a9bd6..aeb95a65 100644 Binary files a/resources/tables/CR_XID_Continue.bin and b/resources/tables/CR_XID_Continue.bin differ diff --git a/resources/tables/CR_XID_Start.bin b/resources/tables/CR_XID_Start.bin index 5ebdf7dd..a2969818 100644 Binary files a/resources/tables/CR_XID_Start.bin and b/resources/tables/CR_XID_Start.bin differ diff --git a/resources/tables/CR_XPosixPunct.bin b/resources/tables/CR_XPosixPunct.bin index 51a7e764..299d6684 100644 Binary files a/resources/tables/CR_XPosixPunct.bin and b/resources/tables/CR_XPosixPunct.bin differ diff --git a/resources/tables/CR_Yezidi.bin b/resources/tables/CR_Yezidi.bin new file mode 100644 index 00000000..ca0dd1b5 Binary files /dev/null and b/resources/tables/CR_Yezidi.bin differ diff --git a/resources/tables/CaseFold.bin b/resources/tables/CaseFold.bin index cff2bb50..99f6fe12 100644 Binary files a/resources/tables/CaseFold.bin and b/resources/tables/CaseFold.bin differ diff --git a/resources/tables/CaseUnfold_11.bin b/resources/tables/CaseUnfold_11.bin index e40a062b..cb09c68c 100644 Binary files a/resources/tables/CaseUnfold_11.bin and b/resources/tables/CaseUnfold_11.bin differ diff --git a/resources/tables/Transcoder_Cesu8_ByteArray.bin b/resources/tables/Transcoder_Cesu8_ByteArray.bin new file mode 100644 index 00000000..fa55e869 Binary files /dev/null and b/resources/tables/Transcoder_Cesu8_ByteArray.bin differ diff --git a/resources/tables/Transcoder_Cesu8_WordArray.bin b/resources/tables/Transcoder_Cesu8_WordArray.bin new file mode 100644 index 00000000..eb72d0e5 Binary files /dev/null and b/resources/tables/Transcoder_Cesu8_WordArray.bin differ diff --git a/resources/tables/Transcoder_Escape_ByteArray.bin b/resources/tables/Transcoder_Escape_ByteArray.bin index a425608e..22cd44c7 100644 Binary files a/resources/tables/Transcoder_Escape_ByteArray.bin and b/resources/tables/Transcoder_Escape_ByteArray.bin differ diff --git a/resources/tables/Transcoder_Escape_WordArray.bin b/resources/tables/Transcoder_Escape_WordArray.bin index 0d705633..e0ca9d15 100644 Binary files a/resources/tables/Transcoder_Escape_WordArray.bin and b/resources/tables/Transcoder_Escape_WordArray.bin differ diff --git a/resources/tables/Transcoder_SingleByte_ByteArray.bin b/resources/tables/Transcoder_SingleByte_ByteArray.bin index 2d5dd43e..284b334d 100644 Binary files a/resources/tables/Transcoder_SingleByte_ByteArray.bin and b/resources/tables/Transcoder_SingleByte_ByteArray.bin differ diff --git a/resources/tables/Transcoder_SingleByte_WordArray.bin b/resources/tables/Transcoder_SingleByte_WordArray.bin index 538b7f29..9ccb6072 100644 Binary files a/resources/tables/Transcoder_SingleByte_WordArray.bin and b/resources/tables/Transcoder_SingleByte_WordArray.bin differ diff --git a/scripts/UnicodeCodeRangeTemplate.java b/scripts/UnicodeCodeRangeTemplate.java new file mode 100644 index 00000000..4ec427a1 --- /dev/null +++ b/scripts/UnicodeCodeRangeTemplate.java @@ -0,0 +1,52 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.jcodings.unicode; + +import org.jcodings.util.ArrayReader; +import org.jcodings.CodeRange; + +public enum UnicodeCodeRange { +%{extcrs}; + + private final String table; + final byte[]name; + private int[]range; + + private UnicodeCodeRange(String name, String table) { + this.table = table; + this.name = name.getBytes(); + } + + int[]getRange() { + if (range == null) range = ArrayReader.readIntArray(table); + return range; + } + + public boolean contains(int code) { + return CodeRange.isInCodeRange(range, code); + } + + public int getCType() { + return ordinal(); + } + + static final UnicodeCodeRange[]CodeRangeTable = UnicodeCodeRange.values(); + static final int MAX_WORD_LENGTH = %{max_length}; +} diff --git a/scripts/generate.rb b/scripts/generate.rb index 5e785f86..6970aebc 100644 --- a/scripts/generate.rb +++ b/scripts/generate.rb @@ -1,8 +1,14 @@ +#!/usr/bin/env ruby # coding: utf-8 -REPO_PATH = ARGV.first || '/usr/src/ruby-2.5.0' # path to ruby repo -SECTION_NAME = "rdata" -UNICODE_VERSION = "10.0.0" +REPO_PATH = ARGV.first || '/usr/src/ruby' # path to ruby repo +SECTION_NAME, G_PREFIX = case RUBY_PLATFORM +when /linux/i; ["rodata", ""] +when /darwin/i; ["const_data", "g"] +else ;["rdata", ""] +end + +UNICODE_VERSION = "17.0.0" SRC_DIR = "../src/org/jcodings" DST_BIN_DIR = "../resources/tables" INDENT = " " * 4 @@ -18,8 +24,8 @@ def generate_data def process_binary obj_name binary = open(obj_name, "rb"){|f|f.read} - offset = `objdump -h -j .#{SECTION_NAME} #{obj_name}`[/\.#{SECTION_NAME}.*?(\w+)\s+\S+$/, 1].to_i(16) - `nm --no-sort --defined-only #{obj_name}`.split("\n").map{|s|s.split(/\s+/)}.each do |address, _, name| + offset = `#{G_PREFIX}objdump -h -j .#{SECTION_NAME} #{obj_name}`[/\.#{SECTION_NAME}.*?(\w+)\s+\S+$/, 1].to_i(16) + `#{G_PREFIX}nm --no-sort --defined-only #{obj_name}`.split("\n").map{|s|s.split(/\s+/)}.each do |address, _, name| yield name, binary, address.to_i(16) + offset end end @@ -59,6 +65,7 @@ def generate_encoding_list "KOI8-R" => "KOI8R", "KOI8-U" => "KOI8U", "Shift_JIS" => "SJIS", + "CESU-8" => "CESU8", "UTF-16BE" => "UTF16BE", "UTF-16LE" => "UTF16LE", "UTF-32BE" => "UTF32BE", @@ -74,6 +81,7 @@ def generate_encoding_list defines, other = open("#{REPO_PATH}/encdb.h").read.tr('()', '').scan(/ENC_([A-Z_]+)(.*?);/m).partition { |a, b| a =~ /DEFINE/ } other << ["ALIAS", "\"MS932\", \"Windows-31J\""] + other << ["ALIAS", "\"UTF8\", \"UTF-8\""] open("#{SRC_DIR}/EncodingList.java", "wb") { |f| f << open("EncodingListTemplate.java", "rb").read. sub(/%\{defines\}/, defines.map { |cmd, name| "#{INDENT*2}EncodingDB.declare(#{name}, \"#{enc_map[name[/[^"]+/]] || (raise 'class not found for encoding ' + name)}\");" }.join("\n")). @@ -84,9 +92,10 @@ def generate_encoding_list def generate_transcoder_list generic_list = [] + specific_list = [] transcoder_list = [] - Dir["#{REPO_PATH}/enc/trans/*.c"].reject{|f| f =~ /transdb/}.each do |trans_file| + Dir["#{REPO_PATH}/enc/trans/*.c"].reject{|f| f =~ /transdb/}.sort.each do |trans_file| name = trans_file[/(\w+)\.c/, 1].split('_').map{|e| e.capitalize}.join("") trans_src = open(trans_file){|f|f.read} @@ -97,10 +106,14 @@ def generate_transcoder_list body = body.gsub(/(\/\*.*?\*\/)/, "").split(',').map{|e|e.strip} src, dst, tree_start, table_info, iul, max_in, max_out, conv, state_size, state_init, state_fini, *funcs = body tree_start = trans_src[/#define\s+#{tree_start}\s+WORDINDEX2INFO\((\d+)\)/, 1].to_i << 2 - state_size = "0" if state_size == "sizeof(struct from_utf8_mac_status)" - generic = funcs.all?{|f|f == "NULL"} + state_size = "24" if state_size == "sizeof(struct from_utf8_mac_status)" + generic = funcs.all?{|f|f == "NULL" || f == "0"} - generic_list << [src, dst, tree_start, "\"#{name}\"", iul, max_in, max_out, "AsciiCompatibility.#{conv.split('_').last.upcase}", state_size] if generic + if generic + generic_list << [src, dst, tree_start, "\"#{name}\"", iul, max_in, max_out, "AsciiCompatibility.#{conv.split('_').last.upcase}", state_size] + else + specific_list << [t_name, src, dst, tree_start, "\"#{name}\"", iul, max_in, max_out, "AsciiCompatibility.#{conv.split('_').last.upcase}", state_size] + end transcoder_list << [src, dst, t_name, !generic] end @@ -110,26 +123,41 @@ def generate_transcoder_list sub(/%\{generic\}/, generic_list.map{|g| "#{INDENT*2}new GenericTranscoderEntry(#{g.join(', ')})"}.join(",\n")). sub(/%\{switch\}/, transcoder_list.map{|src, dst, cls, specific| "#{INDENT*3}case \"#{cls}\": return #{cls}_Transcoder.INSTANCE;" if specific}.compact.join("\n")) + generate_specific_transcoder_data(specific_list) +end + +def generate_specific_transcoder_data(specific_list) + specific_list.each do |transcoder| + path = "#{SRC_DIR}/transcode/specific/#{transcoder[0]}_Transcoder.java" + next unless File.exist?(path) + + content = open(path, "rb"){ |f| f.read } + if content.match("super\\((.*)\\);") + content.sub!($1, transcoder[1..-1].join(", ")) + open(path, "wb") << content + end + end end def generate_transoder_data - Dir["#{REPO_PATH}/enc/trans/*.c"].reject{|f| f =~ /transdb/}.each do |trans_file| + Dir["#{REPO_PATH}/enc/trans/*.c"].reject{|f| f =~ /transdb/}.sort.each do |trans_file| # next unless trans_file =~ /utf8/ trans_file = trans_file[/(.*)\./, 1] src = open("#{trans_file}.c", "rb").read + make_name = -> (name) {name.split('_').map{|e|e.capitalize}.join('')} process_binary "#{trans_file}.o" do |name, binary, address| case name when /(.*)_byte_array/ name = $1 size = src[/(\w+?_byte_array)\[(\d+?)\]/m, 2].to_i - open("#{DST_BIN_DIR}/" + "Transcoder_#{name.capitalize.tr('_', '')}_ByteArray.bin", "wb") do |f| + open("#{DST_BIN_DIR}/" + "Transcoder_#{make_name.(name)}_ByteArray.bin", "wb") do |f| f << [size].pack("N") f << binary[address, size] end when /(.*)_word_array/ name = $1 size = src[/(\w+?_word_array)\[(\d+?)\]/m, 2].to_i - open("#{DST_BIN_DIR}/" + "Transcoder_#{name.capitalize.tr('_', '')}_WordArray.bin", "wb") do |f| + open("#{DST_BIN_DIR}/" + "Transcoder_#{make_name.(name)}_WordArray.bin", "wb") do |f| f << [size].pack("N") address.step(address + (size * 4 - 1), 4).each do |adr| f << binary[adr, 4].unpack("l").pack("N") @@ -182,8 +210,8 @@ def generate_coderange_list end.flatten(1) max_length = out.max_by{|name, table|name.length}.first.length.to_s - open("#{SRC_DIR}/unicode/UnicodeProperties.java", "wb") do |f| f << - open("UnicodePropertiesTemplate.java", "rb").read.sub(/%\{max_length\}/, max_length).sub(/%\{extcrs\}/, out.map{|name, table| "#{INDENT * 2}" + "new CodeRangeEntry(\"#{name}\", \"CR_#{table}\")"}.join(",\n")) + open("#{SRC_DIR}/unicode/UnicodeCodeRange.java", "wb") do |f| f << + open("UnicodeCodeRangeTemplate.java", "rb").read.sub(/%\{max_length\}/, max_length).sub(/%\{extcrs\}/, out.map{|name, table| "#{INDENT}" + "#{name.upcase.tr('=.', '_')}(\"#{name}\", \"CR_#{table}\")"}.join(",\n")) end end diff --git a/src/org/jcodings/AbstractEncoding.java b/src/org/jcodings/AbstractEncoding.java index 82df4342..d8393558 100644 --- a/src/org/jcodings/AbstractEncoding.java +++ b/src/org/jcodings/AbstractEncoding.java @@ -85,7 +85,6 @@ public void applyAllCaseFold(int flag, ApplyAllCaseFoldFunction fun, Object arg) asciiApplyAllCaseFold(flag, fun, arg); } - protected static final CaseFoldCodeItem[] EMPTY_FOLD_CODES = new CaseFoldCodeItem[]{}; protected final CaseFoldCodeItem[]asciiCaseFoldCodesByString(int flag, byte[]bytes, int p, int end) { int b = bytes[p] & 0xff; @@ -94,7 +93,7 @@ public void applyAllCaseFold(int flag, ApplyAllCaseFoldFunction fun, Object arg) } else if (0x61 <= b && b <= 0x7a) { return new CaseFoldCodeItem[]{CaseFoldCodeItem.create(1, b - 0x20)}; } else { - return EMPTY_FOLD_CODES; + return CaseFoldCodeItem.EMPTY_FOLD_CODES; } } diff --git a/src/org/jcodings/CaseFoldCodeItem.java b/src/org/jcodings/CaseFoldCodeItem.java index d1802c26..753240b8 100644 --- a/src/org/jcodings/CaseFoldCodeItem.java +++ b/src/org/jcodings/CaseFoldCodeItem.java @@ -20,6 +20,8 @@ package org.jcodings; public final class CaseFoldCodeItem { + public static final CaseFoldCodeItem[] EMPTY_FOLD_CODES = new CaseFoldCodeItem[]{}; + public final int byteLen; public final int code[]; diff --git a/src/org/jcodings/CaseFoldMapEncoding.java b/src/org/jcodings/CaseFoldMapEncoding.java index 3cd03fd4..80ff4e20 100644 --- a/src/org/jcodings/CaseFoldMapEncoding.java +++ b/src/org/jcodings/CaseFoldMapEncoding.java @@ -106,7 +106,7 @@ private void ssApplyAllCaseFold(int flag, ApplyAllCaseFoldFunction fun, Object a } } } - return EMPTY_FOLD_CODES; + return CaseFoldCodeItem.EMPTY_FOLD_CODES; } @Override diff --git a/src/org/jcodings/CodeRange.java b/src/org/jcodings/CodeRange.java index b8f177c9..c6b2d38e 100644 --- a/src/org/jcodings/CodeRange.java +++ b/src/org/jcodings/CodeRange.java @@ -19,7 +19,7 @@ */ package org.jcodings; -public class CodeRange { +public final class CodeRange { public static boolean isInCodeRange(int[]p, int code) { return isInCodeRange(p, 0, code); } diff --git a/src/org/jcodings/Config.java b/src/org/jcodings/Config.java index 06cd208a..c880f69f 100644 --- a/src/org/jcodings/Config.java +++ b/src/org/jcodings/Config.java @@ -62,4 +62,12 @@ public interface Config { final int ENC_CASE_FOLD_MIN = INTERNAL_ENC_CASE_FOLD_MULTI_CHAR; final int ENC_CASE_FOLD_DEFAULT = ENC_CASE_FOLD_MIN; + final String UNICODE_VERSION_STRING = "17.0.0"; + final int UNICODE_VERSION_MAJOR = 17; + final int UNICODE_VERSION_MINOR = 0; + final int UNICODE_VERSION_TEENY = 0; + final String UNICODE_EMOJI_VERSION_STRING = "17.0"; + final int UNICODE_EMOJI_VERSION_MAJOR = 17; + final int UNICODE_EMOJI_VERSION_MINOR = 0; + } diff --git a/src/org/jcodings/Encoding.java b/src/org/jcodings/Encoding.java index 02a93d24..c537e144 100644 --- a/src/org/jcodings/Encoding.java +++ b/src/org/jcodings/Encoding.java @@ -24,6 +24,7 @@ import org.jcodings.ascii.AsciiTables; import org.jcodings.constants.CharacterType; import org.jcodings.exception.EncodingException; +import org.jcodings.exception.EncodingError; import org.jcodings.exception.ErrorMessages; import org.jcodings.exception.InternalException; import org.jcodings.util.BytesHash; @@ -147,7 +148,7 @@ Encoding replicate(byte[]name) { clone.index = count++; return clone; } catch (CloneNotSupportedException cnse){ - throw new EncodingException(ErrorMessages.ERR_COULD_NOT_REPLICATE, new String(name)); + throw new EncodingException(EncodingError.ERR_COULD_NOT_REPLICATE, new String(name)); } } @@ -564,7 +565,11 @@ public final boolean isFixedWidth() { public static final byte NEW_LINE = (byte)0x0a; public static Encoding load(String name) { - String encClassName = name; + return load(name, "org.jcodings.specific"); + } + + public static Encoding load(String name, String pkg) { + String encClassName = pkg + "." + name + "Encoding"; Class encClass; try { encClass = Class.forName(encClassName); diff --git a/src/org/jcodings/EncodingDB.java b/src/org/jcodings/EncodingDB.java index 78650ec7..a3b7b2e5 100644 --- a/src/org/jcodings/EncodingDB.java +++ b/src/org/jcodings/EncodingDB.java @@ -23,7 +23,7 @@ import org.jcodings.exception.InternalException; import org.jcodings.util.CaseInsensitiveBytesHash; -public class EncodingDB { +public final class EncodingDB { public static final class Entry { private static int count; diff --git a/src/org/jcodings/EncodingList.java b/src/org/jcodings/EncodingList.java index 11a69ec3..35e64f84 100644 --- a/src/org/jcodings/EncodingList.java +++ b/src/org/jcodings/EncodingList.java @@ -28,12 +28,12 @@ static final void load() { EncodingDB.declare("Big5", "BIG5"); EncodingDB.declare("Big5-HKSCS", "Big5HKSCS"); EncodingDB.declare("Big5-UAO", "Big5UAO"); + EncodingDB.declare("CESU-8", "CESU8"); EncodingDB.declare("CP949", "CP949"); EncodingDB.declare("Emacs-Mule", "EmacsMule"); EncodingDB.declare("EUC-JP", "EUCJP"); EncodingDB.declare("EUC-KR", "EUCKR"); EncodingDB.declare("EUC-TW", "EUCTW"); - EncodingDB.declare("GB2312", "GB2312"); EncodingDB.declare("GB18030", "GB18030"); EncodingDB.declare("GBK", "GBK"); EncodingDB.declare("ISO-8859-1", "ISO8859_1"); @@ -69,6 +69,8 @@ static final void load() { EncodingDB.alias("BINARY", "ASCII-8BIT"); EncodingDB.replicate("IBM437", "ASCII-8BIT"); EncodingDB.alias("CP437", "IBM437"); + EncodingDB.replicate("IBM720", "ASCII-8BIT"); + EncodingDB.alias("CP720", "IBM720"); EncodingDB.replicate("IBM737", "ASCII-8BIT"); EncodingDB.alias("CP737", "IBM737"); EncodingDB.replicate("IBM775", "ASCII-8BIT"); @@ -126,6 +128,7 @@ static final void load() { EncodingDB.alias("EUC-JISX0213", "EUC-JIS-2004" /* defined at JIS X 0213:2000, and obsolete at JIS X 0213:2004 */); EncodingDB.alias("eucKR", "EUC-KR"); EncodingDB.alias("eucTW", "EUC-TW"); + EncodingDB.replicate("GB2312", "EUC-KR"); EncodingDB.alias("EUC-CN", "GB2312"); EncodingDB.alias("eucCN", "GB2312"); EncodingDB.replicate("GB12345", "GB2312"); @@ -194,6 +197,7 @@ static final void load() { EncodingDB.replicate("UTF8-SoftBank", "UTF-8"); EncodingDB.replicate("SJIS-SoftBank", "Windows-31J"); EncodingDB.alias("MS932", "Windows-31J"); + EncodingDB.alias("UTF8", "UTF-8"); } public static Encoding getInstance(String name) { @@ -204,12 +208,12 @@ public static Encoding getInstance(String name) { case "BIG5": return BIG5Encoding.INSTANCE; case "Big5HKSCS": return Big5HKSCSEncoding.INSTANCE; case "Big5UAO": return Big5UAOEncoding.INSTANCE; + case "CESU8": return CESU8Encoding.INSTANCE; case "CP949": return CP949Encoding.INSTANCE; case "EmacsMule": return EmacsMuleEncoding.INSTANCE; case "EUCJP": return EUCJPEncoding.INSTANCE; case "EUCKR": return EUCKREncoding.INSTANCE; case "EUCTW": return EUCTWEncoding.INSTANCE; - case "GB2312": return GB2312Encoding.INSTANCE; case "GB18030": return GB18030Encoding.INSTANCE; case "GBK": return GBKEncoding.INSTANCE; case "ISO8859_1": return ISO8859_1Encoding.INSTANCE; diff --git a/src/org/jcodings/MultiByteEncoding.java b/src/org/jcodings/MultiByteEncoding.java index daab9de9..f2d7d857 100644 --- a/src/org/jcodings/MultiByteEncoding.java +++ b/src/org/jcodings/MultiByteEncoding.java @@ -22,6 +22,8 @@ import org.jcodings.ascii.AsciiTables; import org.jcodings.exception.ErrorCodes; +import static java.lang.Integer.toUnsignedLong; + public abstract class MultiByteEncoding extends AbstractEncoding { protected final int EncLen[]; @@ -184,7 +186,7 @@ protected final int mb4CodeToMbc(int code, byte[]bytes, int p) { } protected final boolean mb2IsCodeCType(int code, int ctype) { - if ((code & 0xFFFFFFFFL) < 128) { // unsigned 32-bit + if (toUnsignedLong(code) < 128) { // unsigned 32-bit return isCodeCTypeInternal(code, ctype); // configured with ascii } else { if (isWordGraphPrint(ctype)) { @@ -199,6 +201,7 @@ protected final boolean mb4IsCodeCType(int code, int ctype) { } /* onigenc_strlen */ + @Override public int strLength(byte[]bytes, int p, int end) { int n = 0; int q = p; @@ -209,6 +212,7 @@ public int strLength(byte[]bytes, int p, int end) { return n; } + @Override public int strCodeAt(byte[]bytes, int p, int end, int index) { int n = 0; int q = p; @@ -219,4 +223,8 @@ public int strCodeAt(byte[]bytes, int p, int end, int index) { } return -1; } + + public static boolean isInRange(int code, int from, int to) { + return code - from >= 0 && to - code >= 0; + } } diff --git a/src/org/jcodings/exception/CharacterPropertyException.java b/src/org/jcodings/exception/CharacterPropertyException.java index 59c660a2..f628d10e 100644 --- a/src/org/jcodings/exception/CharacterPropertyException.java +++ b/src/org/jcodings/exception/CharacterPropertyException.java @@ -21,7 +21,9 @@ public class CharacterPropertyException extends EncodingException { - public CharacterPropertyException(EncodingError error) { + private static final long serialVersionUID = 1L; + + public CharacterPropertyException(EncodingError error) { super(error); } diff --git a/src/org/jcodings/exception/EncodingException.java b/src/org/jcodings/exception/EncodingException.java index 592cae8a..f85d8d32 100644 --- a/src/org/jcodings/exception/EncodingException.java +++ b/src/org/jcodings/exception/EncodingException.java @@ -20,6 +20,9 @@ package org.jcodings.exception; public class EncodingException extends JCodingsException { + + private static final long serialVersionUID = 1L; + private final EncodingError error; public EncodingException(EncodingError error) { diff --git a/src/org/jcodings/exception/JCodingsException.java b/src/org/jcodings/exception/JCodingsException.java index bd42cc49..a8d77a90 100644 --- a/src/org/jcodings/exception/JCodingsException.java +++ b/src/org/jcodings/exception/JCodingsException.java @@ -20,6 +20,9 @@ package org.jcodings.exception; public class JCodingsException extends RuntimeException { + + private static final long serialVersionUID = 1L; + public JCodingsException(String message) { super(message); } diff --git a/src/org/jcodings/exception/TranscoderException.java b/src/org/jcodings/exception/TranscoderException.java index a4beea83..dda56cdd 100644 --- a/src/org/jcodings/exception/TranscoderException.java +++ b/src/org/jcodings/exception/TranscoderException.java @@ -20,6 +20,9 @@ package org.jcodings.exception; public class TranscoderException extends JCodingsException { + + private static final long serialVersionUID = 1L; + public TranscoderException(String message) { super(message); } diff --git a/src/org/jcodings/specific/BaseEUCJPEncoding.java b/src/org/jcodings/specific/BaseEUCJPEncoding.java index 41e6857a..f5390ce5 100644 --- a/src/org/jcodings/specific/BaseEUCJPEncoding.java +++ b/src/org/jcodings/specific/BaseEUCJPEncoding.java @@ -62,6 +62,17 @@ public int codeToMbc(int code, byte[]bytes, int p) { return p_ - p; } + private static int getLowerCase(int code) { + if (isInRange(code, 0xa3c1, 0xa3da)) { + return code + 0x0020; + } else if (isInRange(code, 0xa6a1, 0xa6b8)) { + return code + 0x0020; + } else if (isInRange(code, 0xa7a1, 0xa7c1)) { + return code + 0x0030; + } + return code; + } + @Override public int mbcCaseFold(int flag, byte[]bytes, IntHolder pp, int end, byte[]lower) { int p = pp.value; @@ -72,15 +83,16 @@ public int mbcCaseFold(int flag, byte[]bytes, IntHolder pp, int end, byte[]lower pp.value++; return 1; } else { - int len = length(bytes, p, end); - for (int i=0; i 0xfe - 0xa1; } diff --git a/src/org/jcodings/specific/BaseSJISEncoding.java b/src/org/jcodings/specific/BaseSJISEncoding.java index 7a35ddd1..980ea292 100644 --- a/src/org/jcodings/specific/BaseSJISEncoding.java +++ b/src/org/jcodings/specific/BaseSJISEncoding.java @@ -72,9 +72,29 @@ public int codeToMbc(int code, byte[]bytes, int p) { return p_ - p; } + private static int getLowerCase(int code) { + if (isInRange(code, 0x8260, 0x8279)) { + return code + 0x0021; + } else if (isInRange(code, 0x839f, 0x83b6)) { + return code + 0x0020; + } else if (isInRange(code, 0x8440, 0x8460)) { + int d = code >= 0x844f ? 1 : 0; + return code + (0x0030 + d); + } + return code; + } + @Override public int mbcCaseFold(int flag, byte[]bytes, IntHolder pp, int end, byte[]lower) { - return mbnMbcCaseFold(flag, bytes, pp, end, lower); + if (isAscii(bytes[pp.value])) { + return asciiMbcCaseFold(flag, bytes, pp, end, lower); + } else { + int lowerP = 0; + int code = getLowerCase(mbcToCode(bytes, pp.value, end)); + int len = codeToMbc(code, lower, lowerP); + pp.value += len; + return len; + } } private static final int CR_Hiragana[] = { diff --git a/src/org/jcodings/specific/BaseUTF8Encoding.java b/src/org/jcodings/specific/BaseUTF8Encoding.java index 625662a8..6969571d 100644 --- a/src/org/jcodings/specific/BaseUTF8Encoding.java +++ b/src/org/jcodings/specific/BaseUTF8Encoding.java @@ -25,6 +25,8 @@ import org.jcodings.exception.ErrorCodes; import org.jcodings.unicode.UnicodeEncoding; +import static java.lang.Integer.toUnsignedLong; + abstract class BaseUTF8Encoding extends UnicodeEncoding { static final boolean USE_INVALID_CODE_SCHEME = true; @@ -72,7 +74,7 @@ public int codeToMbcLength(int code) { return 2; } else if ((code & 0xffff0000) == 0) { return 3; - } else if ((code & 0xFFFFFFFFL) <= VALID_CODE_LIMIT) { + } else if (toUnsignedLong(code) <= VALID_CODE_LIMIT) { return 4; } else if (USE_INVALID_CODE_SCHEME && code == INVALID_CODE_FE) { return 1; @@ -125,7 +127,7 @@ public int codeToMbc(int code, byte[]bytes, int p) { } else if ((code & 0xffff0000) == 0) { bytes[p_++] = (byte)(((code >>> 12) & 0x0f) | 0xe0); bytes[p_++] = trailS(code, 6); - } else if ((code & 0xFFFFFFFFL) <= VALID_CODE_LIMIT) { + } else if (toUnsignedLong(code) <= VALID_CODE_LIMIT) { bytes[p_++] = (byte)(((code >>> 18) & 0x07) | 0xf0); bytes[p_++] = trailS(code, 12); bytes[p_++] = trailS(code, 6); diff --git a/src/org/jcodings/specific/CESU8Encoding.java b/src/org/jcodings/specific/CESU8Encoding.java new file mode 100644 index 00000000..c4e4c1a3 --- /dev/null +++ b/src/org/jcodings/specific/CESU8Encoding.java @@ -0,0 +1,447 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.jcodings.specific; + +import org.jcodings.Config; +import org.jcodings.IntHolder; +import org.jcodings.ascii.AsciiTables; +import org.jcodings.exception.ErrorCodes; +import org.jcodings.unicode.UnicodeEncoding; + +import static java.lang.Integer.toUnsignedLong; + +public final class CESU8Encoding extends UnicodeEncoding { + static final boolean USE_INVALID_CODE_SCHEME = true; + + protected CESU8Encoding() { + super("CESU-8", 1, 6, CESU8EncLen, CESU8Trans); + } + + @Override + public String getCharsetName() { + return "CESU-8"; + } + + @Override + public int length(byte[] bytes, int p, int end) { + int b = bytes[p] & 0xff; + if (b <= 127) { + return 1; + } + int s = TransZero[b]; + if (s < 0) + return CHAR_INVALID; + return lengthForOneUptoSix(bytes, p, end, b, s); + } + + private int lengthForOneUptoSix(byte[] bytes, int p, int end, int b, int s) { + if (++p == end) { + return missing(b, 1); + } + s = Trans[s][bytes[p] & 0xff]; + if (s < 0) { + return s == A ? 2 : CHAR_INVALID; + } + if (++p == end) { + return missing(b, s == 4 ? 4 : TransZero[b] - 2); + } + s = Trans[s][bytes[p] & 0xff]; + if (s < 0) { + return s == A ? 3 : CHAR_INVALID; + } + if (++p == end) + return missing(b, 3); + s = Trans[s][bytes[p] & 0xff]; + if (s < 0) { + return s == A ? 4 : CHAR_INVALID; + } + if (++p == end) + return missing(b, 2); + s = Trans[s][bytes[p] & 0xff]; + if (s < 0) { + return s == A ? 5 : CHAR_INVALID; + } + if (++p == end) + return missing(b, 1); + s = Trans[s][bytes[p] & 0xff]; + return s == A ? 6 : CHAR_INVALID; + } + + @Override + public boolean isNewLine(byte[] bytes, int p, int end) { + if (p < end) { + if (bytes[p] == (byte) 0x0a) + return true; + + if (Config.USE_UNICODE_ALL_LINE_TERMINATORS) { + if (!Config.USE_CRNL_AS_LINE_TERMINATOR) { + if (bytes[p] == (byte) 0x0d) + return true; + } + + if (p + 1 < end) { + if (bytes[p + 1] == (byte) 0x85 && bytes[p] == (byte) 0xc2) + return true; + if (p + 2 < end) { + if ((bytes[p + 2] == (byte) 0xa8 || bytes[p + 2] == (byte) 0xa9) && + bytes[p + 1] == (byte) 0x80 && bytes[p] == (byte) 0xe2) + return true; + } + } + } + } + return false; + } + + private static final int INVALID_CODE_FE = 0xfffffffe; + private static final int INVALID_CODE_FF = 0xffffffff; + private static final int VALID_CODE_LIMIT = 0x0010ffff; + + @Override + public int codeToMbcLength(int code) { + if ((code & 0xffffff80) == 0) { + return 1; + } else if ((code & 0xfffff800) == 0) { + return 2; + } else if ((code & 0xffff0000) == 0) { + return 3; + } else if (toUnsignedLong(code) <= VALID_CODE_LIMIT) { + return 6; + } else if (USE_INVALID_CODE_SCHEME && code == INVALID_CODE_FE) { + return 1; + } else if (USE_INVALID_CODE_SCHEME && code == INVALID_CODE_FF) { + return 1; + } else { + return ErrorCodes.ERR_TOO_BIG_WIDE_CHAR_VALUE; + } + } + + @Override + public int mbcToCode(byte[] bytes, int p, int end) { + int len = length(bytes, p, end); + int c = bytes[p] & 0xff; + + switch (len) { + case 1: + return c; + case 2: + return ((c & 0x1F) << 6) | (bytes[p + 1] & 0xff & 0x3f); + case 3: + return utf8Decode3ByteSequence(bytes, p, c); + case 6: { + int high = utf8Decode3ByteSequence(bytes, p, c); + int low = utf8Decode3ByteSequence(bytes, p + 3); + return ((high & 0x03ff) << 10) + (low & 0x03ff) + 0x10000; + } + } + + if (USE_INVALID_CODE_SCHEME) { + if (c > 0xfd) { + return ((c == 0xfe) ? INVALID_CODE_FE : INVALID_CODE_FF); + } + } + return c; + } + + private static int utf8Decode3ByteSequence(byte[] bytes, int p) { + return utf8Decode3ByteSequence(bytes, p, bytes[p]); + } + + private static int utf8Decode3ByteSequence(byte[] bytes, int p, int c) { + return ((c & 0xF) << 12) | ((bytes[p + 1] & 0xff & 0x3f) << 6) | (bytes[p + 2] & 0xff & 0x3f); + } + + static byte trailS(int code, int shift) { + return (byte) (((code >>> shift) & 0x3f) | 0x80); + } + + static byte trail0(int code) { + return (byte) ((code & 0x3f) | 0x80); + } + + static byte trailS(long code, int shift) { + return (byte) (((code >>> shift) & 0x3f) | 0x80); + } + + static byte trail0(long code) { + return (byte) ((code & 0x3f) | 0x80); + } + + @Override + public int codeToMbc(int code, byte[] bytes, int p) { + int p_ = p; + if ((code & 0xffffff80) == 0) { + bytes[p_] = (byte) code; + return 1; + } else { + if ((code & 0xfffff800) == 0) { + bytes[p_++] = (byte) (((code >>> 6) & 0x1f) | 0xc0); + } else if ((code & 0xffff0000) == 0) { + bytes[p_++] = (byte) (((code >>> 12) & 0x0f) | 0xe0); + bytes[p_++] = trailS(code, 6); + } else if (toUnsignedLong(code) <= VALID_CODE_LIMIT) { + long high = (code >> 10) + 0xD7C0; + code = (code & 0x3FF) + 0xDC00; + bytes[p_++] = (byte) (((high >> 12) & 0x0f) | 0xe0); + bytes[p_++] = trailS(high, 6); + bytes[p_++] = trail0(high); + bytes[p_++] = (byte) (((code >> 12) & 0x0f) | 0xe0); + bytes[p_++] = trailS(code, 6); + } else if (USE_INVALID_CODE_SCHEME && code == INVALID_CODE_FE) { + bytes[p_] = (byte) 0xfe; + return 1; + } else if (USE_INVALID_CODE_SCHEME && code == INVALID_CODE_FF) { + bytes[p_] = (byte) 0xff; + return 1; + } else { + return ErrorCodes.ERR_TOO_BIG_WIDE_CHAR_VALUE; + } + bytes[p_++] = trail0(code); + return p_ - p; + } + } + + @Override + public int mbcCaseFold(int flag, byte[] bytes, IntHolder pp, int end, byte[] fold) { + int p = pp.value; + int foldP = 0; + + if (isMbcAscii(bytes[p])) { + + if (Config.USE_UNICODE_CASE_FOLD_TURKISH_AZERI) { + if ((flag & Config.CASE_FOLD_TURKISH_AZERI) != 0) { + if (bytes[p] == (byte) 0x49) { + fold[foldP++] = (byte) 0xc4; + fold[foldP] = (byte) 0xb1; + pp.value++; + return 2; + } + } + } + + fold[foldP] = AsciiTables.ToLowerCaseTable[bytes[p] & 0xff]; + pp.value++; + return 1; + } else { + return super.mbcCaseFold(flag, bytes, pp, end, fold); + } + } + + @Override + public int[] ctypeCodeRange(int ctype, IntHolder sbOut) { + sbOut.value = 0x80; + return super.ctypeCodeRange(ctype); + } + + private static boolean utf8IsLead(int c) { + return ((c & 0xc0) & 0xff) != 0x80; + } + + @Override + public int leftAdjustCharHead(byte[] bytes, int p, int s, int end) { + if (s <= p) + return s; + int p_ = s; + while (!utf8IsLead(bytes[p_] & 0xff) && p_ > p) + p_--; + if (p_ > p && s - p_ == 2 && Character.isLowSurrogate((char) utf8Decode3ByteSequence(bytes, p_))) { + int pSurrogatePair = p_ - 1; + while (!utf8IsLead(bytes[pSurrogatePair] & 0xff) && pSurrogatePair > p) + pSurrogatePair--; + if (p_ - pSurrogatePair == 3 && Character.isHighSurrogate((char) utf8Decode3ByteSequence(bytes, pSurrogatePair))) { + return pSurrogatePair; + } + } + return p_; + } + + @Override + public boolean isReverseMatchAllowed(byte[] bytes, int p, int end) { + return true; + } + + private static final int CESU8EncLen[] = { + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 + }; + + static final int CESU8Trans[][] = new int[][] { + { /* S0 0 1 2 3 4 5 6 7 8 9 a b c d e f */ + /* 0 */ A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, + /* 1 */ A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, + /* 2 */ A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, + /* 3 */ A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, + /* 4 */ A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, + /* 5 */ A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, + /* 6 */ A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, + /* 7 */ A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, + /* 8 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, + /* 9 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, + /* a */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, + /* b */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, + /* c */ F, F, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + /* d */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + /* e */ 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 3, 3, + /* f */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F + }, + { /* S1 0 1 2 3 4 5 6 7 8 9 a b c d e f */ + /* 0 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, + /* 1 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, + /* 2 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, + /* 3 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, + /* 4 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, + /* 5 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, + /* 6 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, + /* 7 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, + /* 8 */ A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, + /* 9 */ A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, + /* a */ A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, + /* b */ A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, A, + /* c */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, + /* d */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, + /* e */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, + /* f */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F + }, + { /* S2 0 1 2 3 4 5 6 7 8 9 a b c d e f */ + /* 0 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, + /* 1 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, + /* 2 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, + /* 3 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, + /* 4 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, + /* 5 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, + /* 6 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, + /* 7 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, + /* 8 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, + /* 9 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, + /* a */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + /* b */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + /* c */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, + /* d */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, + /* e */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, + /* f */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F + }, + { /* S3 0 1 2 3 4 5 6 7 8 9 a b c d e f */ + /* 0 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, + /* 1 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, + /* 2 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, + /* 3 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, + /* 4 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, + /* 5 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, + /* 6 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, + /* 7 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, + /* 8 */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + /* 9 */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + /* a */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + /* b */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + /* c */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, + /* d */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, + /* e */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, + /* f */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F + }, + { /* S4 0 1 2 3 4 5 6 7 8 9 a b c d e f */ + /* 0 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, + /* 1 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, + /* 2 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, + /* 3 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, + /* 4 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, + /* 5 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, + /* 6 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, + /* 7 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, + /* 8 */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + /* 9 */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + /* a */ 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, + /* b */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, + /* c */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, + /* d */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, + /* e */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, + /* f */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F + }, + { /* S5 0 1 2 3 4 5 6 7 8 9 a b c d e f */ + /* 0 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, + /* 1 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, + /* 2 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, + /* 3 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, + /* 4 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, + /* 5 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, + /* 6 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, + /* 7 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, + /* 8 */ 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, + /* 9 */ 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, + /* a */ 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, + /* b */ 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, + /* c */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, + /* d */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, + /* e */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, + /* f */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F + }, + { /* S6 0 1 2 3 4 5 6 7 8 9 a b c d e f */ + /* 0 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, + /* 1 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, + /* 2 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, + /* 3 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, + /* 4 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, + /* 5 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, + /* 6 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, + /* 7 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, + /* 8 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, + /* 9 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, + /* a */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, + /* b */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, + /* c */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, + /* d */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, + /* e */ F, F, F, F, F, F, F, F, F, F, F, F, F, 7, F, F, + /* f */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F + }, + { /* S7 0 1 2 3 4 5 6 7 8 9 a b c d e f */ + /* 0 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, + /* 1 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, + /* 2 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, + /* 3 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, + /* 4 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, + /* 5 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, + /* 6 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, + /* 7 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, + /* 8 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, + /* 9 */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, + /* a */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, + /* b */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + /* c */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, + /* d */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, + /* e */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, + /* f */ F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F + } + }; + + public static final CESU8Encoding INSTANCE = new CESU8Encoding(); +} diff --git a/src/org/jcodings/specific/EUCKREncoding.java b/src/org/jcodings/specific/EUCKREncoding.java index 924fbe42..8247ae54 100644 --- a/src/org/jcodings/specific/EUCKREncoding.java +++ b/src/org/jcodings/specific/EUCKREncoding.java @@ -69,6 +69,7 @@ public boolean isCodeCType(int code, int ctype) { } // euckr_islead + @Override protected boolean isLead(int c) { return ((c) < 0xa1 || (c) == 0xff); } diff --git a/src/org/jcodings/specific/EmacsMuleEncoding.java b/src/org/jcodings/specific/EmacsMuleEncoding.java index a3198dbc..165ccac6 100644 --- a/src/org/jcodings/specific/EmacsMuleEncoding.java +++ b/src/org/jcodings/specific/EmacsMuleEncoding.java @@ -99,7 +99,7 @@ public int leftAdjustCharHead(byte[] bytes, int p, int s, int end) { int p_ = s; while (!islead(bytes[p_] & 0xff) && p_ > p) p_--; return p_; - }; + } @Override public boolean isReverseMatchAllowed(byte[]bytes, int p, int end) { diff --git a/src/org/jcodings/specific/GB18030Encoding.java b/src/org/jcodings/specific/GB18030Encoding.java index e2f9cc10..e68ed67d 100644 --- a/src/org/jcodings/specific/GB18030Encoding.java +++ b/src/org/jcodings/specific/GB18030Encoding.java @@ -120,7 +120,7 @@ private enum State { Odd_CM_Odd_C4CM, Even_CM_Even_C4CM, Odd_CM_Even_C4CM - }; + } @Override public int leftAdjustCharHead(byte[]bytes, int start, int s, int end) { diff --git a/src/org/jcodings/specific/ISO8859_1Encoding.java b/src/org/jcodings/specific/ISO8859_1Encoding.java index b7b0b3de..2f182584 100644 --- a/src/org/jcodings/specific/ISO8859_1Encoding.java +++ b/src/org/jcodings/specific/ISO8859_1Encoding.java @@ -85,7 +85,7 @@ public void applyAllCaseFold(int flag, ApplyAllCaseFoldFunction fun, Object arg) return new CaseFoldCodeItem[]{CaseFoldCodeItem.create(1, b - 0x20)}; } } - return EMPTY_FOLD_CODES; + return CaseFoldCodeItem.EMPTY_FOLD_CODES; } @Override diff --git a/src/org/jcodings/specific/UTF32BEEncoding.java b/src/org/jcodings/specific/UTF32BEEncoding.java index af112732..cc5fe277 100644 --- a/src/org/jcodings/specific/UTF32BEEncoding.java +++ b/src/org/jcodings/specific/UTF32BEEncoding.java @@ -36,8 +36,8 @@ public boolean isNewLine(byte[]bytes, int p, int end) { if (bytes[p + 3] == (byte)0x0a && bytes[p + 2] == 0 && bytes[p + 1] == 0 && bytes[p] == 0) return true; if (Config.USE_UNICODE_ALL_LINE_TERMINATORS) { - if ((Config.USE_CRNL_AS_LINE_TERMINATOR && bytes[p + 3] == (byte)0x0d) || - bytes[p + 3] == (byte)0x85 && bytes[p + 2] == 0 && bytes[p + 1] == 0 && bytes[p] == 0) return true; + if ((bytes[p + 3] == (byte)0x0b || bytes[p + 3] == (byte)0x0c || bytes[p + 3] == (byte)0x0d || bytes[p + 3] == (byte)0x85) + && bytes[p + 2] == 0 && bytes[p + 1] == 0 && bytes[p] == 0) return true; if (bytes[p + 2] == (byte)0x20 && (bytes[p + 3] == (byte)0x29 || bytes[p + 3] == (byte)0x28) && diff --git a/src/org/jcodings/specific/UTF32LEEncoding.java b/src/org/jcodings/specific/UTF32LEEncoding.java index 826d5234..8ad28064 100644 --- a/src/org/jcodings/specific/UTF32LEEncoding.java +++ b/src/org/jcodings/specific/UTF32LEEncoding.java @@ -33,15 +33,14 @@ protected UTF32LEEncoding() { @Override public boolean isNewLine(byte[]bytes, int p, int end) { if (p + 3 < end) { - if (bytes[p] == (byte)0x0a && bytes[p + 1] == 0 && bytes[p + 2] == 0 && bytes[p + 3] == 0) return true; + if (bytes[p + 3] == 0 && bytes[p + 2] == 0 && bytes[p + 1] == 0 && bytes[p] == (byte)0x0a) return true; if (Config.USE_UNICODE_ALL_LINE_TERMINATORS) { - if ((Config.USE_CRNL_AS_LINE_TERMINATOR && bytes[p] == (byte)0x0d) || - bytes[p] == (byte)0x85 && bytes[p + 1] == 0 && bytes[p + 2] == 0 && bytes[3] == 0) return true; + if (bytes[p + 3] == 0 && bytes[p + 2] == 0 && bytes[p + 1] == 0 && + (bytes[p] == (byte)0x0b || bytes[p] == (byte)0x0c || bytes[p] == (byte)0x0d || bytes[p] == (byte)0x85)) return true; - if (bytes[p + 1] == (byte)0x20 && - (bytes[p] == (byte)0x29 || bytes[p] == (byte)0x28) && - bytes[p + 2] == 0 && bytes[p + 3] == 0) return true; + if (bytes[p + 3] == 0 && bytes[p + 2] == 0 && bytes[p + 1] == (byte)0x20 && + (bytes[p] == (byte)0x29 || bytes[p] == (byte)0x28)) return true; } // USE_UNICODE_ALL_LINE_TERMINATORS } return false; @@ -68,10 +67,12 @@ public int mbcCaseFold(int flag, byte[]bytes, IntHolder pp, int end, byte[]fold) int foldP = 0; if (isAscii(bytes[p] & 0xff) && bytes[p + 1] == 0 && bytes[p + 2] == 0 && bytes[p + 3] == 0) { - if (Config.USE_UNICODE_CASE_FOLD_TURKISH_AZERI && (flag & Config.CASE_FOLD_TURKISH_AZERI) != 0) { - if (bytes[p] == (byte)0x49) { - fold[foldP++] = (byte)0x31; - fold[foldP] = (byte)0x01; + if (Config.USE_UNICODE_CASE_FOLD_TURKISH_AZERI) { + if ((flag & Config.CASE_FOLD_TURKISH_AZERI) != 0) { + if (bytes[p] == (byte)0x49) { + fold[foldP++] = (byte)0x31; + fold[foldP++] = (byte)0x01; + } } } else { fold[foldP++] = AsciiTables.ToLowerCaseTable[bytes[p] & 0xff]; diff --git a/src/org/jcodings/specific/Windows_1251Encoding.java b/src/org/jcodings/specific/Windows_1251Encoding.java index 4f4f0e21..007ec1d6 100644 --- a/src/org/jcodings/specific/Windows_1251Encoding.java +++ b/src/org/jcodings/specific/Windows_1251Encoding.java @@ -21,7 +21,6 @@ import org.jcodings.CaseFoldMapEncoding; import org.jcodings.Config; -import org.jcodings.ISOEncoding; import org.jcodings.IntHolder; import org.jcodings.constants.CharacterType; diff --git a/src/org/jcodings/spi/Charsets.java b/src/org/jcodings/spi/Charsets.java index beec0462..7e739312 100644 --- a/src/org/jcodings/spi/Charsets.java +++ b/src/org/jcodings/spi/Charsets.java @@ -10,10 +10,13 @@ */ public class Charsets extends java.nio.charset.spi.CharsetProvider { private static final List charsets = Collections.singletonList((Charset) ISO_8859_16.INSTANCE); + + @Override public Iterator charsets() { return charsets.iterator(); } + @Override public Charset charsetForName(String charsetName) { if ("ISO-8859-16".equals(charsetName) || ISO_8859_16.INSTANCE.aliases().contains(charsetName)) { return ISO_8859_16.INSTANCE; diff --git a/src/org/jcodings/spi/ISO_8859_16.java b/src/org/jcodings/spi/ISO_8859_16.java index d9a209ef..e4c50592 100644 --- a/src/org/jcodings/spi/ISO_8859_16.java +++ b/src/org/jcodings/spi/ISO_8859_16.java @@ -20,6 +20,7 @@ package org.jcodings.spi; +import java.nio.Buffer; import java.nio.ByteBuffer; import java.nio.CharBuffer; import java.nio.charset.Charset; @@ -27,6 +28,8 @@ import java.nio.charset.CharsetEncoder; import java.nio.charset.CoderResult; +import static java.lang.Byte.toUnsignedInt; + /** * Implementation of Charset, CharsetDecoder, and CharsetEncoder for ISO-8859-16. */ @@ -62,7 +65,7 @@ protected CoderResult decodeLoop(ByteBuffer in, CharBuffer out) { for (;;) { if (!in.hasRemaining()) return CoderResult.UNDERFLOW; if (!out.hasRemaining()) return CoderResult.OVERFLOW; - int b = in.get() & 0xFF; + int b = toUnsignedInt(in.get()); char c = TABLE[b]; out.put(c); } @@ -174,7 +177,7 @@ protected CoderResult encodeLoop(CharBuffer in, ByteBuffer out) { if (replace != null) { if (out.remaining() < replace.length) { - in.position(in.position() - 1); + ((Buffer) in).position(in.position() - 1); return CoderResult.OVERFLOW; } else { out.put(replace); diff --git a/src/org/jcodings/transcode/EConv.java b/src/org/jcodings/transcode/EConv.java index 86176c8f..269b4da0 100644 --- a/src/org/jcodings/transcode/EConv.java +++ b/src/org/jcodings/transcode/EConv.java @@ -395,7 +395,7 @@ private EConvResult convertInternal(byte[] in, Ptr inPtr, int inStop, byte[] out Ptr inDataStartPtr = new Ptr(inBuf.dataStart); res = transConv(inBuf.bytes, inDataStartPtr, inBuf.dataEnd, out, outPtr, outStop, (flags & ~AFTER_OUTPUT) | PARTIAL_INPUT, resultPosition); inBuf.dataStart = inDataStartPtr.p; - if (!res.isSourceBufferEmpty()) return convertInternalResult(EConvResult.SourceBufferEmpty, resultPosition); + if (!res.isSourceBufferEmpty()) return convertInternalResult(res, resultPosition); } if (hasOutput && (flags & AFTER_OUTPUT) != 0 && inPtr.p != inStop) { @@ -732,6 +732,11 @@ public void binmode() { if (entry.getTranscoder() != null) transcoders[n++] = entry.getTranscoder(); } + if ((flags & LF_NEWLINE_DECORATOR) != 0) { + TranscoderDB.Entry entry = TranscoderDB.getEntry(NULL_STRING, "lf_newline".getBytes()); + if (entry.getTranscoder() != null) transcoders[n++] = entry.getTranscoder(); + } + int nTrans = numTranscoders; int j = 0; for (int i = 0; i < nTrans; i++) { @@ -851,7 +856,6 @@ public boolean equals(Object other) { EConv ec2 = (EConv)other; int i; - if (ec2 == null) return false; if (ec1.source != ec2.source && !Arrays.equals(ec1.source, ec2.source)) return false; diff --git a/src/org/jcodings/transcode/EConvFlags.java b/src/org/jcodings/transcode/EConvFlags.java index bbd6f7af..51743e22 100644 --- a/src/org/jcodings/transcode/EConvFlags.java +++ b/src/org/jcodings/transcode/EConvFlags.java @@ -29,23 +29,24 @@ public interface EConvFlags { final int UNDEF_REPLACE = 0x00000020; final int UNDEF_HEX_CHARREF = 0x00000030; - final int DECORATOR_MASK = 0x0000ff00; - final int NEWLINE_DECORATOR_MASK = 0x00003f00; + final int DECORATOR_MASK = 0x0001ff00; + final int NEWLINE_DECORATOR_MASK = 0x00007f00; final int NEWLINE_DECORATOR_READ_MASK = 0x00000f00; - final int NEWLINE_DECORATOR_WRITE_MASK = 0x00003000; + final int NEWLINE_DECORATOR_WRITE_MASK = 0x00007000; final int UNIVERSAL_NEWLINE_DECORATOR = 0x00000100; final int CRLF_NEWLINE_DECORATOR = 0x00001000; final int CR_NEWLINE_DECORATOR = 0x00002000; - final int XML_TEXT_DECORATOR = 0x00004000; - final int XML_ATTR_CONTENT_DECORATOR = 0x00008000; + final int LF_NEWLINE_DECORATOR = 0x00004000; + final int XML_TEXT_DECORATOR = 0x00008000; + final int XML_ATTR_CONTENT_DECORATOR = 0x00010000; final int STATEFUL_DECORATOR_MASK = 0x00f00000; final int XML_ATTR_QUOTE_DECORATOR = 0x00100000; - final int PARTIAL_INPUT = 0x00010000; - final int AFTER_OUTPUT = 0x00020000; + final int PARTIAL_INPUT = 0x00020000; + final int AFTER_OUTPUT = 0x00040000; final int MAX_ECFLAGS_DECORATORS = 32; } \ No newline at end of file diff --git a/src/org/jcodings/transcode/TranscodeFunctions.java b/src/org/jcodings/transcode/TranscodeFunctions.java index 4b5c0e73..c892d698 100644 --- a/src/org/jcodings/transcode/TranscodeFunctions.java +++ b/src/org/jcodings/transcode/TranscodeFunctions.java @@ -4,6 +4,9 @@ import java.util.Arrays; +import static java.lang.Byte.toUnsignedInt; +import static java.lang.Integer.toUnsignedLong; + /** * Created by headius on 3/4/14. */ @@ -11,6 +14,27 @@ public class TranscodeFunctions { public static final int BE = 1; public static final int LE = 2; + public static int funSoToCESU8(byte[] statep, byte[] s, int sStart, int l, byte[] o, int oStart, int osize) { + long scalar = ((toUnsignedInt(s[sStart+0]) & 0x07) << 18) | ((toUnsignedInt(s[sStart+1]) & 0x3F) << 12) | ((toUnsignedInt(s[sStart+2]) & 0x3F) << 6) | (toUnsignedInt(s[sStart+3]) & 0x3F); + scalar -= 0x10000; + o[oStart+0] = (byte)0xED; + o[oStart+1] = (byte)(0xA0 | (scalar >> 16)); + o[oStart+2] = (byte)(0x80 | ((scalar >> 10) & 0x3F)); + o[oStart+3] = (byte)0xED; + o[oStart+4] = (byte)(0xB0 | ((scalar >> 6) & 0x0F)); + o[oStart+5] = (byte)(0x80 | (scalar & 0x3F)); + return 6; + } + + public static int funSoFromCESU8(byte[] statep, byte[] s, int sStart, int l, byte[] o, int oStart, int osize) { + long scalar = (((toUnsignedInt(s[sStart+1]) & 0x0F) << 16) | ((toUnsignedInt(s[sStart+2]) & 0x3F) << 10) | ((toUnsignedInt(s[sStart+4]) & 0x0F) << 6) | (toUnsignedInt(s[sStart+5]) & 0x3F)) + 0x10000; + o[oStart+0] = (byte)(0xF0 | (scalar >> 18)); + o[oStart+1] = (byte)(0x80 | ((scalar >> 12) & 0x3F)); + o[oStart+2] = (byte)(0x80 | ((scalar >> 6) & 0x3F)); + o[oStart+3] = (byte)(0x80 | (scalar & 0x3F)); + return 4; + } + public static int funSoToUTF16(byte[] statep, byte[] sBytes, int sStart, int l, byte[] o, int oStart, int osize) { int sp = 0; if (statep[sp] == 0) { @@ -23,7 +47,7 @@ public static int funSoToUTF16(byte[] statep, byte[] sBytes, int sStart, int l, } public static int funSoToUTF16BE(byte[] statep, byte[] s, int sStart, int l, byte[] o, int oStart, int osize) { - int s0 = s[sStart] & 0xFF; + int s0 = toUnsignedInt(s[sStart]); int s1, s2, s3; if ((s0 & 0x80) == 0) { @@ -31,20 +55,20 @@ public static int funSoToUTF16BE(byte[] statep, byte[] s, int sStart, int l, byt o[oStart + 1] = (byte)s0; return 2; } else if ((s0 & 0xE0) == 0xC0) { - s1 = s[sStart+1] & 0xFF; + s1 = toUnsignedInt(s[sStart+1]); o[oStart] = (byte)((s0 >> 2) & 0x07); o[oStart + 1] = (byte)(((s0 & 0x03) << 6) | (s1 & 0x3F)); return 2; } else if ((s0 & 0xF0) == 0xE0) { - s1 = s[sStart+1] & 0xFF; - s2 = s[sStart+2] & 0xFF; + s1 = toUnsignedInt(s[sStart+1]); + s2 = toUnsignedInt(s[sStart+2]); o[oStart] = (byte)((s0 << 4) | ((s1 >> 2) ^ 0x20)); o[oStart + 1] = (byte)((s1 << 6) | (s2 ^ 0x80)); return 2; } else { - s1 = s[sStart+1] & 0xFF; - s2 = s[sStart+2] & 0xFF; - s3 = s[sStart+3] & 0xFF; + s1 = toUnsignedInt(s[sStart+1]); + s2 = toUnsignedInt(s[sStart+2]); + s3 = toUnsignedInt(s[sStart+3]); int w = (((s0 & 0x07) << 2) | ((s1 >> 4) & 0x03)) - 1; o[oStart] = (byte)(0xD8 | (w >> 2)); o[oStart + 1] = (byte)((w << 6) | ((s1 & 0x0F) << 2) | ((s2 >> 4) - 8)); @@ -55,27 +79,27 @@ public static int funSoToUTF16BE(byte[] statep, byte[] s, int sStart, int l, byt } public static int funSoToUTF16LE(byte[] statep, byte[] s, int sStart, int l, byte[] o, int oStart, int osize) { - int s0 = s[sStart] & 0xFF; + int s0 = toUnsignedInt(s[sStart]); int s1; if ((s0 & 0x80) == 0) { o[oStart + 1] = (byte)0x00; o[oStart] = (byte)s0; return 2; } else if ((s0 & 0xE0) == 0xC0) { - s1 = s[sStart+1] & 0xFF; + s1 = toUnsignedInt(s[sStart+1]); o[oStart + 1] = (byte)((s0 >> 2) & 0x07); o[oStart] = (byte)(((s0 & 0x03) << 6) | (s1 & 0x3F)); return 2; } else if ((s0 & 0xF0) == 0xE0) { - s1 = s[sStart+1] & 0xFF; - int s2 = s[sStart+2] & 0xFF; + s1 = toUnsignedInt(s[sStart+1]); + int s2 = toUnsignedInt(s[sStart+2]); o[oStart + 1] = (byte)((s0 << 4) | ((s1 >> 2) ^ 0x20)); o[oStart] = (byte)((s1 << 6) | (s2 ^ 0x80)); return 2; } else { - s1 = s[sStart+1] & 0xFF; - int s2 = s[sStart+2] & 0xFF; - int s3 = s[sStart+3] & 0xFF; + s1 = toUnsignedInt(s[sStart+1]); + int s2 = toUnsignedInt(s[sStart+2]); + int s3 = toUnsignedInt(s[sStart+3]); int w = (((s0 & 0x07) << 2) | ((s1 >> 4) & 0x03)) - 1; o[oStart + 1] = (byte)(0xD8 | (w >> 2)); o[oStart] = (byte)((w << 6) | ((s1 & 0x0F) << 2) | ((s2 >> 4) - 8)); @@ -99,27 +123,27 @@ public static int funSoToUTF32(byte[] statep, byte[] s, int sStart, int l, byte[ } public static int funSoToUTF32BE(byte[] statep, byte[] s, int sStart, int l, byte[] o, int oStart, int osize) { - int s0 = s[sStart] & 0xFF; + int s0 = toUnsignedInt(s[sStart]); int s1, s2, s3; o[oStart] = 0; if ((s0 & 0x80) == 0) { o[oStart + 1] = o[oStart + 2] = 0x00; o[oStart + 3] = (byte)s0; } else if ((s0 & 0xE0) == 0xC0) { - s1 = s[sStart+1] & 0xFF; + s1 = toUnsignedInt(s[sStart+1]); o[oStart + 1] = 0x00; o[oStart + 2] = (byte)((s0 >> 2) & 0x07); o[oStart + 3] = (byte)(((s0 & 0x03) << 6) | (s1 & 0x3F)); } else if ((s0 & 0xF0) == 0xE0) { - s1 = s[sStart+1] & 0xFF; - s2 = s[sStart+2] & 0xFF; + s1 = toUnsignedInt(s[sStart+1]); + s2 = toUnsignedInt(s[sStart+2]); o[oStart + 1] = 0x00; o[oStart + 2] = (byte)((s0 << 4) | ((s1 >> 2) ^ 0x20)); o[oStart + 3] = (byte)((s1 << 6) | (s2 ^ 0x80)); } else { - s1 = s[sStart+1] & 0xFF; - s2 = s[sStart+2] & 0xFF; - s3 = s[sStart+3] & 0xFF; + s1 = toUnsignedInt(s[sStart+1]); + s2 = toUnsignedInt(s[sStart+2]); + s3 = toUnsignedInt(s[sStart+3]); o[oStart + 1] = (byte)(((s0 & 0x07) << 2) | ((s1 >> 4) & 0x03)); o[oStart + 2] = (byte)(((s1 & 0x0F) << 4) | ((s2 >> 2) & 0x0F)); o[oStart + 3] = (byte)(((s2 & 0x03) << 6) | (s3 & 0x3F)); @@ -129,25 +153,25 @@ public static int funSoToUTF32BE(byte[] statep, byte[] s, int sStart, int l, byt public static int funSoToUTF32LE(byte[] statep, byte[] s, int sStart, int l, byte[] o, int oStart, int osize) { o[oStart+3] = 0; - int s0 = s[sStart] & 0xFF; + int s0 = toUnsignedInt(s[sStart]); if ((s0 & 0x80) == 0) { o[oStart + 2] = o[oStart+1] = 0x00; o[oStart] = (byte)s0; } else if ((s[sStart] & 0xE0) == 0xC0) { - int s1 = s[sStart+1] & 0xFF; + int s1 = toUnsignedInt(s[sStart+1]); o[oStart + 2] = 0x00; o[oStart + 1] = (byte)((s0 >> 2) & 0x07); o[oStart] = (byte)(((s0 & 0x03) << 6) | (s1 & 0x3F)); } else if ((s[sStart] & 0xF0) == 0xE0) { - int s1 = s[sStart+1] & 0xFF; - int s2 = s[sStart+2] & 0xFF; + int s1 = toUnsignedInt(s[sStart+1]); + int s2 = toUnsignedInt(s[sStart+2]); o[oStart + 2] = 0x00; o[oStart + 1] = (byte)((s0 << 4) | ((s1 >> 2) ^ 0x20)); o[oStart] = (byte)((s1 << 6) | (s2 ^ 0x80)); } else { - int s1 = s[sStart+1] & 0xFF; - int s2 = s[sStart+2] & 0xFF; - int s3 = s[sStart+3] & 0xFF; + int s1 = toUnsignedInt(s[sStart+1]); + int s2 = toUnsignedInt(s[sStart+2]); + int s3 = toUnsignedInt(s[sStart+3]); o[oStart + 2] = (byte)(((s0 & 0x07) << 2) | ((s1 >> 4) & 0x03)); o[oStart + 1] = (byte)(((s1 & 0x0F) << 4) | ((s2 >> 2) & 0x0F)); o[oStart] = (byte)(((s2 & 0x03) << 6) | (s3 & 0x3F)); @@ -156,15 +180,15 @@ public static int funSoToUTF32LE(byte[] statep, byte[] s, int sStart, int l, byt } public static int funSiFromUTF32(byte[] statep, byte[] s, int sStart, int l) { - int s0 = s[sStart] & 0xFF; - int s1 = s[sStart+1] & 0xFF; - int s2 = s[sStart+2] & 0xFF; + int s0 = toUnsignedInt(s[sStart]); + int s1 = toUnsignedInt(s[sStart+1]); + int s2 = toUnsignedInt(s[sStart+2]); int s3; byte[] sp = statep; switch (sp[0]) { case 0: - s3 = s[sStart+3] & 0xFF; + s3 = toUnsignedInt(s[sStart+3]); if (s0 == 0 && s1 == 0 && s2 == 0xFE && s3 == 0xFF) { sp[0] = BE; return TranscodingInstruction.ZERObt; @@ -180,7 +204,7 @@ public static int funSiFromUTF32(byte[] statep, byte[] s, int sStart, int l) { } break; case LE: - s3 = s[sStart+3] & 0xFF; + s3 = toUnsignedInt(s[sStart+3]); if (s3 == 0 && ((0 < s2 && s2 <= 0x10) || (s2 == 0 && (s1 < 0xD8 || 0xDF < s1)))) return TranscodingInstruction.FUNso; @@ -200,9 +224,9 @@ public static int funSoFromUTF32(byte[] statep, byte[] s, int sStart, int l, byt } public static int funSoFromUTF32BE(byte[] statep, byte[] s, int sStart, int l, byte[] o, int oStart, int osize) { - int s1 = s[sStart+1] & 0xFF; - int s2 = s[sStart+2] & 0xFF; - int s3 = s[sStart+3] & 0xFF; + int s1 = toUnsignedInt(s[sStart+1]); + int s2 = toUnsignedInt(s[sStart+2]); + int s3 = toUnsignedInt(s[sStart+3]); if (s1 == 0) { if (s2 == 0 && s3 < 0x80) { o[oStart] = (byte)s3; @@ -227,9 +251,9 @@ public static int funSoFromUTF32BE(byte[] statep, byte[] s, int sStart, int l, b } public static int funSoFromUTF32LE(byte[] statep, byte[] s, int sStart, int l, byte[] o, int oStart, int osize) { - int s0 = s[sStart] & 0xFF; - int s1 = s[sStart+1] & 0xFF; - int s2 = s[sStart+2] & 0xFF; + int s0 = toUnsignedInt(s[sStart]); + int s1 = toUnsignedInt(s[sStart+1]); + int s2 = toUnsignedInt(s[sStart+2]); if (s2 == 0) { if (s1 == 0 && s0 < 0x80) { o[oStart] = (byte)s0; @@ -257,13 +281,13 @@ public static int funSoFromUTF32LE(byte[] statep, byte[] s, int sStart, int l, b public static final int from_UTF_16LE_00toFF_D8toDB = Transcoding.WORDINDEX2INFO(5); public static int funSiFromUTF16(byte[] statep, byte[] s, int sStart, int l) { - int s0 = s[sStart] & 0xFF; + int s0 = toUnsignedInt(s[sStart]); int s1; byte[] sp = statep; switch (sp[0]) { case 0: - s1 = s[sStart+1] & 0xFF; + s1 = toUnsignedInt(s[sStart+1]); if (s0 == 0xFE && s1 == 0xFF) { sp[0] = BE; return TranscodingInstruction.ZERObt; @@ -280,7 +304,7 @@ public static int funSiFromUTF16(byte[] statep, byte[] s, int sStart, int l) { } break; case LE: - s1 = s[sStart+1] & 0xFF; + s1 = toUnsignedInt(s[sStart+1]); if (s1 < 0xD8 || 0xDF < s1) { return TranscodingInstruction.FUNso; } else if (s1 <= 0xDB) { @@ -302,8 +326,8 @@ public static int funSoFromUTF16(byte[] statep, byte[] s, int sStart, int l, byt } public static int funSoFromUTF16BE(byte[] statep, byte[] s, int sStart, int l, byte[] o, int oStart, int osize) { - int s0 = s[sStart] & 0xFF; - int s1 = s[sStart+1] & 0xFF; + int s0 = toUnsignedInt(s[sStart]); + int s1 = toUnsignedInt(s[sStart+1]); int s2, s3; if (s0 == 0 && s1 < 0x80) { o[oStart] = (byte)s1; @@ -318,8 +342,8 @@ public static int funSoFromUTF16BE(byte[] statep, byte[] s, int sStart, int l, b o[oStart + 2] = (byte)(0x80 | (s1 & 0x3F)); return 3; } else { - s2 = s[sStart+2] & 0xFF; - s3 = s[sStart+3] & 0xFF; + s2 = toUnsignedInt(s[sStart+2]); + s3 = toUnsignedInt(s[sStart+3]); long u = (((s0 & 0x03) << 2) | (s1 >> 6)) + 1; o[oStart] = (byte)(0xF0 | (u >> 2)); o[oStart + 1] = (byte)(0x80 | ((u & 0x03) << 4) | (((s1 >> 2) & 0x0F))); @@ -330,8 +354,8 @@ public static int funSoFromUTF16BE(byte[] statep, byte[] s, int sStart, int l, b } public static int funSoFromUTF16LE(byte[] statep, byte[] s, int sStart, int l, byte[] o, int oStart, int osize) { - int s0 = s[sStart] & 0xFF; - int s1 = s[sStart+1] & 0xFF; + int s0 = toUnsignedInt(s[sStart]); + int s1 = toUnsignedInt(s[sStart+1]); if (s1 == 0 && s0 < 0x80) { o[oStart] = (byte)s0; return 1; @@ -345,8 +369,8 @@ public static int funSoFromUTF16LE(byte[] statep, byte[] s, int sStart, int l, b o[oStart + 2] = (byte)(0x80 | (s0 & 0x3F)); return 3; } else { - int s2 = s[sStart+2] & 0xFF; - int s3 = s[sStart+3] & 0xFF; + int s2 = toUnsignedInt(s[sStart+2]); + int s3 = toUnsignedInt(s[sStart+3]); long u = (((s1 & 0x03) << 2) | (s0 >> 6)) + 1; o[oStart] = (byte)(0xF0 | (u >> 2)); o[oStart + 1] = (byte)(0x80 | ((u & 0x03) << 4) | ((s0 >> 2) & 0x0F)); @@ -357,8 +381,8 @@ public static int funSoFromUTF16LE(byte[] statep, byte[] s, int sStart, int l, b } public static int funSoEucjp2Sjis(byte[] statep, byte[] s, int sStart, int _l, byte[] o, int oStart, int osize) { - int s0 = s[sStart] & 0xFF; - int s1 = s[sStart+1] & 0xFF; + int s0 = toUnsignedInt(s[sStart]); + int s1 = toUnsignedInt(s[sStart+1]); if (s0 == 0x8e) { o[oStart] = (byte)s1; return 1; @@ -378,7 +402,7 @@ public static int funSoEucjp2Sjis(byte[] statep, byte[] s, int sStart, int _l, b } public static int funSoSjis2Eucjp(byte[] statep, byte[] s, int sStart, int _l, byte[] o, int oStart, int osize) { - int s0 = s[sStart] & 0xFF; + int s0 = toUnsignedInt(s[sStart]); if (_l == 1) { o[oStart] = (byte)0x8E; o[oStart+1] = (byte)s0; @@ -386,7 +410,7 @@ public static int funSoSjis2Eucjp(byte[] statep, byte[] s, int sStart, int _l, b } else { int h, l; h = s0; - l = s[sStart + 1] & 0xFF; + l = Byte.toUnsignedInt(s[sStart+1]); if (0xe0 <= h) { h -= 64; } @@ -404,11 +428,11 @@ public static int funSoSjis2Eucjp(byte[] statep, byte[] s, int sStart, int _l, b public static int funSoFromGB18030(byte[] statep, byte[] s, int sStart, int l, byte[] o, int oStart, int osize) { - long s0 = s[sStart] & 0xFF; - long s1 = s[sStart+1] & 0xFF; - long s2 = s[sStart+2] & 0xFF; - long s3 = s[sStart+3] & 0xFF; - long u = ((s0 - 0x90) * 10 * 126 * 10 + (s1 - 0x30) * 126 * 10 + (s2 - 0x81) * 10 + (s3 - 0x30) + 0x10000) & 0xFFFFFFFFL; + int s0 = toUnsignedInt(s[sStart]); + int s1 = toUnsignedInt(s[sStart+1]); + int s2 = toUnsignedInt(s[sStart+2]); + int s3 = toUnsignedInt(s[sStart+3]); + long u = toUnsignedLong((s0 - 0x90) * 10 * 126 * 10 + (s1 - 0x30) * 126 * 10 + (s2 - 0x81) * 10 + (s3 - 0x30) + 0x10000); o[oStart] = (byte)(0xF0 | (u >>> 18)); o[oStart+1] = (byte)(0x80 | ((u >>> 12) & 0x3F)); o[oStart+2] = (byte)(0x80 | ((u >>> 6) & 0x3F)); @@ -418,17 +442,17 @@ public static int funSoFromGB18030(byte[] statep, byte[] s, int sStart, int l, b public static int funSioFromGB18030(byte[] statep, byte[] s, int sStart, int l, int info, byte[] o, int oStart, int osize) { - long s0 = s[sStart] & 0xFF; - long s1 = s[sStart+1] & 0xFF; - long diff = info >> 8; + int s0 = toUnsignedInt(s[sStart]); + int s1 = toUnsignedInt(s[sStart+1]); + int diff = info >> 8; long u; /* Unicode Scalar Value */ if ((diff & 0x20000) != 0) { /* GB18030 4 bytes */ - long s2 = s[sStart+2] & 0xFF; - long s3 = s[sStart+3] & 0xFF; - u = (((s0 * 10 + s1) * 126 + s2) * 10 + s3 - diff - 0x170000) & 0xFFFFFFFFL; + int s2 = toUnsignedInt(s[sStart+2]); + int s3 = toUnsignedInt(s[sStart+3]); + u = toUnsignedLong(((s0 * 10 + s1) * 126 + s2) * 10 + s3 - diff - 0x170000); } else { /* GB18030 2 bytes */ - u = (s0 * 256 + s1 + 24055 - diff) & 0xFFFFFFFFL; + u = toUnsignedLong(s0 * 256 + s1 + 24055 - diff); } o[oStart] = (byte)(0xE0 | (u >>> 12)); o[oStart+1] = (byte)(0x80 | ((u >>> 6) & 0x3F)); @@ -438,10 +462,10 @@ public static int funSioFromGB18030(byte[] statep, byte[] s, int sStart, int l, public static int funSoToGB18030(byte[] statep, byte[] s, int sStart, int l, byte[] o, int oStart, int osize) { - long s0 = s[sStart] & 0xFF; - long s1 = s[sStart+1] & 0xFF; - long s2 = s[sStart+2] & 0xFF; - long s3 = s[sStart+3] & 0xFF; + long s0 = toUnsignedInt(s[sStart]); + long s1 = toUnsignedInt(s[sStart+1]); + long s2 = toUnsignedInt(s[sStart+2]); + long s3 = toUnsignedInt(s[sStart+3]); long u = ((s0 & 0x07) << 18) | ((s1 & 0x3F) << 12) | ((s2 & 0x3F) << 6) | (s3 & 0x3F); u -= 0x10000; o[oStart+3] = (byte)(0x30 + (u % 10)); @@ -455,9 +479,9 @@ public static int funSoToGB18030(byte[] statep, byte[] s, int sStart, int l, byt public static int funSioToGB18030(byte[] statep, byte[] s, int sStart, int l, int info, byte[] o, int oStart, int osize) { - long s0 = s[sStart] & 0xFF; - long s1 = s[sStart+1] & 0xFF; - long s2 = s[sStart+2] & 0xFF; + long s0 = toUnsignedInt(s[sStart]); + long s1 = toUnsignedInt(s[sStart+1]); + long s2 = toUnsignedInt(s[sStart+2]); long diff = info >>> 8; long u; /* Unicode Scalar Value */ @@ -515,19 +539,17 @@ public static int funSoCp50220Encoder(byte[] statep, byte[] s, int sStart, int l int output0 = oStart; byte[] sp = statep; - if (sp[0] == G0_JISX0201_KATAKANA) { + if (sp[0] == G0_JISX0201_KATAKANA && sp[2] != 0) { int c = sp[2] & 0x7F; int p = (c - 0x21) * 2; + sp[2] = 0; byte[] pBytes = tbl0208; - if (sp[1] == G0_JISX0208_1983) { - o[oStart++] = 0x1B; - o[oStart++] = (byte)'$'; - o[oStart++] = (byte)'B'; - } + oStart = iso2022jp_put_state(sp, o, (int)sp[1], G0_JISX0208_1983, oStart); sp[0] = G0_JISX0208_1983; + o[oStart++] = pBytes[p++]; - s0 = s[sStart] & 0xFF; - s1 = s[sStart+1] & 0xFF; + s0 = toUnsignedInt(s[sStart]); + s1 = toUnsignedInt(s[sStart+1]); if (l == 2 && s0 == 0x8E) { if (s1 == 0xDE) { o[oStart++] = (byte)(pBytes[p] + 1); @@ -540,26 +562,29 @@ public static int funSoCp50220Encoder(byte[] statep, byte[] s, int sStart, int l o[oStart++] = pBytes[p]; } - s0 = s[sStart] & 0xFF; + s0 = toUnsignedInt(s[sStart]); if (l == 2 && s0 == 0x8E) { - s1 = s[sStart+1] & 0xFF; + s1 = toUnsignedInt(s[sStart+1]); int p = (s1 - 0xA1) * 2; - byte[] pBytes = tbl0208; if ((0xA1 <= s1 && s1 <= 0xB5) || (0xC5 <= s1 && s1 <= 0xC9) || (0xCF <= s1 && s1 <= 0xDF)) { - if (sp[0] != G0_JISX0208_1983) { - o[oStart++] = 0x1b; - o[oStart++] = '$'; - o[oStart++] = 'B'; - sp[0] = G0_JISX0208_1983; - } + byte[] pBytes = tbl0208; + oStart = iso2022jp_put_state(sp, o, (int)sp[0], G0_JISX0208_1983, oStart); + o[oStart++] = pBytes[p++]; o[oStart++] = pBytes[p]; return oStart - output0; } - sp[2] = (byte)s1; + if (s1 > 0xDF) { /* undef */ + oStart = iso2022jp_put_state(sp, o, (int)sp[0], G0_JISX0201_KATAKANA, oStart); + o[oStart++] = (byte) (s1 & 0x7f); + sp[2] = 0; + return oStart - output0; + } + + sp[2] = (byte) s1; sp[1] = sp[0]; sp[0] = G0_JISX0201_KATAKANA; return oStart - output0; @@ -577,7 +602,7 @@ public static int funSoCp5022xEncoder(byte[] statep, byte[] s, int sStart, int l if (l == 1) { newstate = G0_ASCII; - } else if ((s[sStart] & 0xFF) == 0x8E) { + } else if (toUnsignedInt(s[sStart]) == 0x8E) { sStart++; l = 1; newstate = G0_JISX0201_KATAKANA; @@ -585,31 +610,14 @@ public static int funSoCp5022xEncoder(byte[] statep, byte[] s, int sStart, int l newstate = G0_JISX0208_1983; } - if (sp[0] != newstate) { - if (newstate == G0_ASCII) { - o[oStart++] = 0x1b; - o[oStart++] = '('; - o[oStart++] = 'B'; - } - else if (newstate == G0_JISX0201_KATAKANA) { - o[oStart++] = 0x1b; - o[oStart++] = '('; - o[oStart++] = 'I'; - } - else { - o[oStart++] = 0x1b; - o[oStart++] = '$'; - o[oStart++] = 'B'; - } - sp[0] = (byte)newstate; - } + oStart = iso2022jp_put_state(sp, o, (int)sp[0], newstate, oStart); - s0 = s[sStart] & 0xFF; + s0 = toUnsignedInt(s[sStart]); if (l == 1) { o[oStart++] = (byte)(s0 & 0x7f); } else { - s1 = s[sStart+1] & 0xFF; + s1 = toUnsignedInt(s[sStart+1]); o[oStart++] = (byte)(s0 & 0x7f); o[oStart++] = (byte)(s1 & 0x7f); } @@ -623,24 +631,18 @@ public static int finishCp50220Encoder(byte[] statep, byte[] o, int oStart, int if (sp[0] == G0_ASCII) return 0; - if (sp[0] == G0_JISX0201_KATAKANA) { + if (sp[0] == G0_JISX0201_KATAKANA && sp[2] != 0) { int c = sp[2] & 0x7F; int p = (c - 0x21) * 2; byte[] pBytes = tbl0208; - if (sp[1] != G0_JISX0208_1983) { - o[oStart++] = 0x1b; - o[oStart++] = '$'; - o[oStart++] = 'B'; - } - sp[0] = G0_JISX0208_1983; + + oStart = iso2022jp_put_state(sp, o, (int)sp[1], G0_JISX0208_1983, oStart); + o[oStart++] = pBytes[p++]; o[oStart++] = pBytes[p]; } - o[oStart++] = 0x1b; - o[oStart++] = '('; - o[oStart++] = 'B'; - sp[0] = G0_ASCII; + oStart = iso2022jp_put_state(sp, o, (int)sp[0], G0_ASCII, oStart); return oStart - output0; } @@ -654,7 +656,7 @@ public static int iso2022jpEncoderResetSequenceSize(byte[] statep) { public static final int iso2022jp_decoder_jisx0208_rest = Transcoding.WORDINDEX2INFO(16); public static int funSiIso50220jpDecoder(byte[] statep, byte[] s, int sStart, int l) { - int s0 = s[sStart] & 0xFF; + int s0 = toUnsignedInt(s[sStart]); byte[] sp = statep; if (sp[0] == G0_ASCII) return TranscodingInstruction.NOMAP; @@ -665,12 +667,12 @@ else if (0x21 <= s0 && s0 <= 0x7e) } public static int funSoIso2022jpDecoder(byte[] statep, byte[] s, int sStart, int l, byte[] o, int oStart, int oSize) { - int s0 = s[sStart] & 0xFF; - int s1 = s[sStart+1] & 0xFF; + int s0 = toUnsignedInt(s[sStart]); + int s1 = toUnsignedInt(s[sStart+1]); byte[] sp = statep; if (s0 == 0x1b) { if (s1 == '(') { - switch (s[sStart+l-1] & 0xFF) { + switch (toUnsignedInt(s[sStart+l-1])) { case 'B': case 'J': sp[0] = G0_ASCII; @@ -715,6 +717,32 @@ public static int funSoEucjpToStatelessIso2022jp(byte[] statep, byte[] s, int sS return 3; } + private static int iso2022jp_put_state(byte[] sp, byte[] o, int oldstate, int newstate, int oStart) { + if (oldstate != newstate) { + o[oStart++] = 0x1b; + switch (newstate) { + case G0_ASCII: + o[oStart++] = '('; + o[oStart++] = 'B'; + break; + case G0_JISX0201_KATAKANA: + o[oStart++] = '('; + o[oStart++] = 'I'; + break; + case G0_JISX0208_1978: + o[oStart++] = '$'; + o[oStart++] = '@'; + break; + default: + o[oStart++] = '$'; + o[oStart++] = 'B'; + break; + } + sp[0] = (byte) newstate; + } + return oStart; + } + public static int funSoIso2022jpEncoder(byte[] statep, byte[] s, int sStart, int l, byte[] o, int oStart, int oSize) { byte[] sp = statep; int output0 = oStart; @@ -722,29 +750,12 @@ public static int funSoIso2022jpEncoder(byte[] statep, byte[] s, int sStart, int if (l == 1) newstate = G0_ASCII; - else if ((s[sStart] & 0xFF) == EMACS_MULE_LEADING_CODE_JISX0208_1978) + else if (toUnsignedInt(s[sStart]) == EMACS_MULE_LEADING_CODE_JISX0208_1978) newstate = G0_JISX0208_1978; else newstate = G0_JISX0208_1983; - if (sp[0] != newstate) { - if (newstate == G0_ASCII) { - o[oStart++] = 0x1b; - o[oStart++] = '('; - o[oStart++] = 'B'; - } - else if (newstate == G0_JISX0208_1978) { - o[oStart++] = 0x1b; - o[oStart++] = '$'; - o[oStart++] = '@'; - } - else { - o[oStart++] = 0x1b; - o[oStart++] = '$'; - o[oStart++] = 'B'; - } - sp[0] = (byte)newstate; - } + oStart = iso2022jp_put_state(sp, o, (int)sp[0], newstate, oStart); if (l == 1) { o[oStart++] = (byte)(s[sStart] & 0x7f); @@ -763,10 +774,8 @@ public static int finishIso2022jpEncoder(byte[] statep, byte[] o, int oStart, in if (sp[0] == G0_ASCII) return 0; - o[oStart++] = 0x1b; - o[oStart++] = '('; - o[oStart++] = 'B'; - sp[0] = G0_ASCII; + + oStart = iso2022jp_put_state(sp, o, (int)sp[0], G0_ASCII, oStart); return oStart - output0; } @@ -774,7 +783,7 @@ public static int finishIso2022jpEncoder(byte[] statep, byte[] o, int oStart, in public static int funSiCp50221Decoder(byte[] statep, byte[] s, int sStart, int l) { byte[] sp = statep; int c; - int s0 = s[sStart] & 0xFF; + int s0 = toUnsignedInt(s[sStart]); switch (sp[0]) { case G0_ASCII: if (0xA1 <= s0 && s0 <= 0xDF) @@ -873,7 +882,7 @@ public static int iso2022jpKddiInit(byte[] statep) { public static final int iso2022jp_kddi_decoder_jisx0208_rest = Transcoding.WORDINDEX2INFO(16); public static int funSiIso2022jpKddiDecoder(byte[] statep, byte[] s, int sStart, int l) { - int s0 = s[sStart] & 0xFF; + int s0 = toUnsignedInt(s[sStart]); byte[] sp = statep; if (sp[0] == G0_ASCII) { return TranscodingInstruction.NOMAP; @@ -885,12 +894,12 @@ public static int funSiIso2022jpKddiDecoder(byte[] statep, byte[] s, int sStart, } public static int funSoIso2022jpKddiDecoder(byte[] statep, byte[] s, int sStart, int l, byte[] o, int oStart, int oSize) { - int s0 = s[sStart] & 0xFF; - int s1 = s[sStart+1] & 0xFF; + int s0 = toUnsignedInt(s[sStart]); + int s1 = toUnsignedInt(s[sStart+1]); byte[] sp = statep; if (s0 == 0x1b) { if (s1 == '(') { - switch (s[sStart+l-1] & 0xFF) { + switch (toUnsignedInt(s[sStart+l-1])) { case 'B': /* US-ASCII */ case 'J': /* JIS X 0201 Roman */ sp[0] = G0_ASCII; @@ -898,7 +907,7 @@ public static int funSoIso2022jpKddiDecoder(byte[] statep, byte[] s, int sStart, } } else { - switch (s[sStart+l-1] & 0xFF) { + switch (toUnsignedInt(s[sStart+l-1])) { case '@': sp[0] = G0_JISX0208_1978; break; @@ -923,7 +932,7 @@ public static int funSoIso2022jpKddiDecoder(byte[] statep, byte[] s, int sStart, } public static int funSoIso2022jpKddiEncoder(byte[] statep, byte[] s, int sStart, int l, byte[] o, int oStart, int oSize) { - int s0 = s[sStart] & 0xFF; + int s0 = toUnsignedInt(s[sStart]); int s1, s2; byte[] sp = statep; int output0 = oStart; @@ -936,31 +945,14 @@ else if (s0 == EMACS_MULE_LEADING_CODE_JISX0208_1978) else newstate = G0_JISX0208_1983; - if (sp[0] != newstate) { - o[oStart++] = 0x1b; - switch (newstate) { - case G0_ASCII: - o[oStart++] = '('; - o[oStart++] = 'B'; - break; - case G0_JISX0208_1978: - o[oStart++] = '$'; - o[oStart++] = '@'; - break; - default: - o[oStart++] = '$'; - o[oStart++] = 'B'; - break; - } - sp[0] = (byte)newstate; - } + oStart = iso2022jp_put_state(sp, o, (int)sp[0], newstate, oStart); if (l == 1) { o[oStart++] = (byte)(s0 & 0x7f); } else { - s1 = s[sStart+1] & 0xFF; - s2 = s[sStart+2] & 0xFF; + s1 = toUnsignedInt(s[sStart+1]); + s2 = toUnsignedInt(s[sStart+2]); o[oStart++] = (byte)(s1 & 0x7f); o[oStart++] = (byte)(s2 & 0x7f); } @@ -975,10 +967,8 @@ public static int finishIso2022jpKddiEncoder(byte[] statep, byte[] o, int oStart if (sp[0] == G0_ASCII) return 0; - o[oStart++] = 0x1b; - o[oStart++] = '('; - o[oStart++] = 'B'; - sp[0] = G0_ASCII; + + oStart = iso2022jp_put_state(sp, o, (int)sp[0], G0_ASCII, oStart); return oStart - output0; } @@ -1097,7 +1087,7 @@ private static int bufOutputChar(byte[] sp, byte[] o, int oStart) { private static int getInfo(int nextInfo, byte[] sp) { int pos = 0; while (pos < bufBytesize(sp)) { - int next_byte = bufAt(sp, pos++) & 0xFF; + int next_byte = toUnsignedInt(bufAt(sp, pos++)); if (next_byte < UTF8MAC_BL_MIN_BYTE(nextInfo) || UTF8MAC_BL_MAX_BYTE(nextInfo) < next_byte) nextInfo = TranscodingInstruction.INVALID; else { @@ -1110,19 +1100,19 @@ private static int getInfo(int nextInfo, byte[] sp) { } public static int UTF8MAC_BL_MIN_BYTE(int nextInfo) { - return From_UTF8_MAC_Transcoder.INSTANCE.byteArray[BL_BASE(nextInfo)] & 0xFF; + return toUnsignedInt(From_UTF8_MAC_Transcoder.INSTANCE.byteArray[BL_BASE(nextInfo)]); } public static int UTF8MAC_BL_MAX_BYTE(int nextInfo) { - return From_UTF8_MAC_Transcoder.INSTANCE.byteArray[BL_BASE(nextInfo) + 1] & 0xFF; + return toUnsignedInt(From_UTF8_MAC_Transcoder.INSTANCE.byteArray[BL_BASE(nextInfo) + 1]); } public static int UTF8MAC_BL_OFFSET(int nextInfo, int b) { - return From_UTF8_MAC_Transcoder.INSTANCE.byteArray[BL_BASE(nextInfo) + 2 + b - UTF8MAC_BL_MIN_BYTE(nextInfo)] & 0xFF; + return toUnsignedInt(From_UTF8_MAC_Transcoder.INSTANCE.byteArray[BL_BASE(nextInfo) + 2 + b - UTF8MAC_BL_MIN_BYTE(nextInfo)]); } public static int UTF8MAC_BL_ACTION(int nextInfo, byte b) { - return From_UTF8_MAC_Transcoder.INSTANCE.intArray[BL_INFO(nextInfo) + UTF8MAC_BL_OFFSET(nextInfo, b & 0xFF)]; + return From_UTF8_MAC_Transcoder.INSTANCE.intArray[BL_INFO(nextInfo) + UTF8MAC_BL_OFFSET(nextInfo, toUnsignedInt(b))]; } private static int BL_BASE(int nextInfo) { @@ -1154,10 +1144,10 @@ private static int bufInt(byte[] statep, int base) { } private static void bufInt(byte[] statep, int base, int val) { - statep[base] = (byte)((val >>> 24) & 0xFF); - statep[base+1] = (byte)((val >>> 16) & 0xFF); - statep[base+2] = (byte)((val >>> 8) & 0xFF); - statep[base+3] = (byte)(val & 0xFF); + statep[base] = (byte) (val >>> 24); + statep[base+1] = (byte) (val >>> 16); + statep[base+2] = (byte) (val >>> 8); + statep[base+3] = (byte) val; } private static int bufBeg(byte[] statep) { @@ -1265,7 +1255,7 @@ public static int universalNewlineInit(byte[] statep) { } public static int funSoUniversalNewline(byte[] statep, byte[] s, int sStart, int l, byte[] o, int oStart, int oSize) { - int s0 = s[sStart] & 0xFF; + int s0 = toUnsignedInt(s[sStart]); byte[] sp = statep; int len; if (s0 == '\n') { diff --git a/src/org/jcodings/transcode/TranscoderDB.java b/src/org/jcodings/transcode/TranscoderDB.java index 1d6b38ae..e9c72d57 100644 --- a/src/org/jcodings/transcode/TranscoderDB.java +++ b/src/org/jcodings/transcode/TranscoderDB.java @@ -219,6 +219,7 @@ private static EConv open0(byte[] source, byte[] destination, int ecflags) { numTrans = searchPath(source, destination, new SearchPathCallback() { int additional = 0; + @Override public void call(byte[] source, byte[] destination, int depth) { if (lentries.p == null) lentries.p = new Entry[depth + 1 + additional]; lentries.p[depth] = getEntry(source, destination); @@ -246,6 +247,7 @@ public static int decoratorNames(int ecflags, byte[][] decorators) { case UNIVERSAL_NEWLINE_DECORATOR: case CRLF_NEWLINE_DECORATOR: case CR_NEWLINE_DECORATOR: + case LF_NEWLINE_DECORATOR: case 0: break; default: @@ -262,6 +264,7 @@ public static int decoratorNames(int ecflags, byte[][] decorators) { if ((ecflags & CRLF_NEWLINE_DECORATOR) != 0) decorators[numDecorators++] = "crlf_newline".getBytes(); if ((ecflags & CR_NEWLINE_DECORATOR) != 0) decorators[numDecorators++] = "cr_newline".getBytes(); + if ((ecflags & LF_NEWLINE_DECORATOR) != 0) decorators[numDecorators++] = "lf_newline".getBytes(); if ((ecflags & UNIVERSAL_NEWLINE_DECORATOR) != 0) decorators[numDecorators++] = "universal_newline".getBytes(); return numDecorators; diff --git a/src/org/jcodings/transcode/TranscoderList.java b/src/org/jcodings/transcode/TranscoderList.java index 5f9f1aa7..235fa14d 100644 --- a/src/org/jcodings/transcode/TranscoderList.java +++ b/src/org/jcodings/transcode/TranscoderList.java @@ -34,6 +34,8 @@ static void load() { TranscoderDB.declare("UTF-8", "CP951", null /*To_CP951*/); TranscoderDB.declare("Big5-UAO", "UTF-8", null /*From_Big5_UAO*/); TranscoderDB.declare("UTF-8", "Big5-UAO", null /*To_Big5_UAO*/); + TranscoderDB.declare("CESU-8", "UTF-8", "From_CESU_8"); + TranscoderDB.declare("UTF-8", "CESU-8", "To_CESU_8"); TranscoderDB.declare("GB2312", "UTF-8", null /*From_GB2312*/); TranscoderDB.declare("GB12345", "UTF-8", null /*From_GB12345*/); TranscoderDB.declare("UTF-8", "GB2312", null /*To_GB2312*/); @@ -99,6 +101,7 @@ static void load() { TranscoderDB.declare("", "universal_newline", "Universal_newline"); TranscoderDB.declare("", "crlf_newline", null /*Crlf_newline*/); TranscoderDB.declare("", "cr_newline", null /*Cr_newline*/); + TranscoderDB.declare("", "lf_newline", "Lf_newline"); TranscoderDB.declare("US-ASCII", "UTF-8", null /*From_US_ASCII*/); TranscoderDB.declare("UTF-8", "US-ASCII", null /*To_US_ASCII*/); TranscoderDB.declare("ASCII-8BIT", "UTF-8", null /*From_ASCII_8BIT*/); @@ -153,10 +156,12 @@ static void load() { TranscoderDB.declare("UTF-8", "WINDOWS-1257", null /*To_WINDOWS_1257*/); TranscoderDB.declare("IBM437", "UTF-8", null /*From_IBM437*/); TranscoderDB.declare("UTF-8", "IBM437", null /*To_IBM437*/); - TranscoderDB.declare("IBM775", "UTF-8", null /*From_IBM775*/); - TranscoderDB.declare("UTF-8", "IBM775", null /*To_IBM775*/); + TranscoderDB.declare("IBM720", "UTF-8", null /*From_IBM720*/); + TranscoderDB.declare("UTF-8", "IBM720", null /*To_IBM720*/); TranscoderDB.declare("IBM737", "UTF-8", null /*From_IBM737*/); TranscoderDB.declare("UTF-8", "IBM737", null /*To_IBM737*/); + TranscoderDB.declare("IBM775", "UTF-8", null /*From_IBM775*/); + TranscoderDB.declare("UTF-8", "IBM775", null /*To_IBM775*/); TranscoderDB.declare("IBM852", "UTF-8", null /*From_IBM852*/); TranscoderDB.declare("UTF-8", "IBM852", null /*To_IBM852*/); TranscoderDB.declare("IBM855", "UTF-8", null /*From_IBM855*/); @@ -171,6 +176,8 @@ static void load() { TranscoderDB.declare("UTF-8", "IBM862", null /*To_IBM862*/); TranscoderDB.declare("IBM863", "UTF-8", null /*From_IBM863*/); TranscoderDB.declare("UTF-8", "IBM863", null /*To_IBM863*/); + TranscoderDB.declare("IBM864", "UTF-8", null /*From_IBM864*/); + TranscoderDB.declare("UTF-8", "IBM864", null /*To_IBM864*/); TranscoderDB.declare("IBM865", "UTF-8", null /*From_IBM865*/); TranscoderDB.declare("UTF-8", "IBM865", null /*To_IBM865*/); TranscoderDB.declare("IBM866", "UTF-8", null /*From_IBM866*/); @@ -260,7 +267,7 @@ static void load() { new GenericTranscoderEntry("UTF8-SoftBank", "SJIS-SoftBank", 84704, "EmojiSjisSoftbank", 1, 4, 2, AsciiCompatibility.CONVERTER, 0), new GenericTranscoderEntry("", "amp_escape", 8, "Escape", 1, 1, 5, AsciiCompatibility.CONVERTER, 0), new GenericTranscoderEntry("", "xml_text_escape", 32, "Escape", 1, 1, 5, AsciiCompatibility.CONVERTER, 0), - new GenericTranscoderEntry("", "xml_attr_content_escape", 60, "Escape", 1, 1, 6, AsciiCompatibility.CONVERTER, 0), + new GenericTranscoderEntry("", "xml_attr_content_escape", 64, "Escape", 1, 1, 6, AsciiCompatibility.CONVERTER, 0), new GenericTranscoderEntry("GBK", "UTF-8", 89284, "Gbk", 1, 2, 1, AsciiCompatibility.CONVERTER, 0), new GenericTranscoderEntry("UTF-8", "GBK", 182912, "Gbk", 1, 4, 2, AsciiCompatibility.CONVERTER, 0), new GenericTranscoderEntry("EUC-JP", "UTF-8", 54488, "JapaneseEuc", 1, 3, 1, AsciiCompatibility.CONVERTER, 0), @@ -335,63 +342,69 @@ static void load() { new GenericTranscoderEntry("UTF-8", "WINDOWS-1257", 24952, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0), new GenericTranscoderEntry("IBM437", "UTF-8", 25476, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0), new GenericTranscoderEntry("UTF-8", "IBM437", 26312, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0), - new GenericTranscoderEntry("IBM775", "UTF-8", 26836, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0), - new GenericTranscoderEntry("UTF-8", "IBM775", 27480, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0), - new GenericTranscoderEntry("IBM737", "UTF-8", 28004, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0), - new GenericTranscoderEntry("UTF-8", "IBM737", 28516, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0), - new GenericTranscoderEntry("IBM852", "UTF-8", 29040, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0), - new GenericTranscoderEntry("UTF-8", "IBM852", 29656, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0), - new GenericTranscoderEntry("IBM855", "UTF-8", 30180, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0), - new GenericTranscoderEntry("UTF-8", "IBM855", 30732, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0), - new GenericTranscoderEntry("IBM857", "UTF-8", 31248, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0), - new GenericTranscoderEntry("UTF-8", "IBM857", 31760, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0), - new GenericTranscoderEntry("IBM860", "UTF-8", 32284, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0), - new GenericTranscoderEntry("UTF-8", "IBM860", 32672, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0), - new GenericTranscoderEntry("IBM861", "UTF-8", 33196, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0), - new GenericTranscoderEntry("UTF-8", "IBM861", 33508, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0), - new GenericTranscoderEntry("IBM862", "UTF-8", 34032, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0), - new GenericTranscoderEntry("UTF-8", "IBM862", 34276, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0), - new GenericTranscoderEntry("IBM863", "UTF-8", 34800, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0), - new GenericTranscoderEntry("UTF-8", "IBM863", 35180, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0), - new GenericTranscoderEntry("IBM865", "UTF-8", 35704, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0), - new GenericTranscoderEntry("UTF-8", "IBM865", 36016, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0), - new GenericTranscoderEntry("IBM866", "UTF-8", 36540, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0), - new GenericTranscoderEntry("UTF-8", "IBM866", 36996, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0), - new GenericTranscoderEntry("IBM869", "UTF-8", 37488, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0), - new GenericTranscoderEntry("UTF-8", "IBM869", 38004, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0), - new GenericTranscoderEntry("MACCROATIAN", "UTF-8", 38528, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0), - new GenericTranscoderEntry("UTF-8", "MACCROATIAN", 39360, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0), - new GenericTranscoderEntry("MACCYRILLIC", "UTF-8", 39884, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0), - new GenericTranscoderEntry("UTF-8", "MACCYRILLIC", 40588, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0), - new GenericTranscoderEntry("MACGREEK", "UTF-8", 41112, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0), - new GenericTranscoderEntry("UTF-8", "MACGREEK", 41812, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0), - new GenericTranscoderEntry("MACICELAND", "UTF-8", 42336, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0), - new GenericTranscoderEntry("UTF-8", "MACICELAND", 43052, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0), - new GenericTranscoderEntry("MACROMAN", "UTF-8", 43576, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0), - new GenericTranscoderEntry("UTF-8", "MACROMAN", 44060, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0), - new GenericTranscoderEntry("MACROMANIA", "UTF-8", 44584, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0), - new GenericTranscoderEntry("UTF-8", "MACROMANIA", 44960, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0), - new GenericTranscoderEntry("MACTURKISH", "UTF-8", 45480, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0), - new GenericTranscoderEntry("UTF-8", "MACTURKISH", 45836, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0), - new GenericTranscoderEntry("MACUKRAINE", "UTF-8", 46360, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0), - new GenericTranscoderEntry("UTF-8", "MACUKRAINE", 46584, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0), - new GenericTranscoderEntry("KOI8-U", "UTF-8", 47108, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0), - new GenericTranscoderEntry("UTF-8", "KOI8-U", 47892, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0), - new GenericTranscoderEntry("KOI8-R", "UTF-8", 48416, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0), - new GenericTranscoderEntry("UTF-8", "KOI8-R", 48948, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0), - new GenericTranscoderEntry("TIS-620", "UTF-8", 49312, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0), - new GenericTranscoderEntry("UTF-8", "TIS-620", 49356, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0), - new GenericTranscoderEntry("CP850", "UTF-8", 49880, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0), - new GenericTranscoderEntry("UTF-8", "CP850", 50428, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0), - new GenericTranscoderEntry("CP852", "UTF-8", 29040, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0), - new GenericTranscoderEntry("UTF-8", "CP852", 29656, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0), - new GenericTranscoderEntry("CP855", "UTF-8", 30180, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0), - new GenericTranscoderEntry("UTF-8", "CP855", 30732, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0), + new GenericTranscoderEntry("IBM720", "UTF-8", 26808, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0), + new GenericTranscoderEntry("UTF-8", "IBM720", 27288, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0), + new GenericTranscoderEntry("IBM737", "UTF-8", 27812, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0), + new GenericTranscoderEntry("UTF-8", "IBM737", 28300, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0), + new GenericTranscoderEntry("IBM775", "UTF-8", 28824, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0), + new GenericTranscoderEntry("UTF-8", "IBM775", 29468, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0), + new GenericTranscoderEntry("IBM852", "UTF-8", 29992, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0), + new GenericTranscoderEntry("UTF-8", "IBM852", 30608, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0), + new GenericTranscoderEntry("IBM855", "UTF-8", 31132, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0), + new GenericTranscoderEntry("UTF-8", "IBM855", 31684, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0), + new GenericTranscoderEntry("IBM857", "UTF-8", 32200, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0), + new GenericTranscoderEntry("UTF-8", "IBM857", 32712, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0), + new GenericTranscoderEntry("IBM860", "UTF-8", 33236, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0), + new GenericTranscoderEntry("UTF-8", "IBM860", 33624, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0), + new GenericTranscoderEntry("IBM861", "UTF-8", 34148, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0), + new GenericTranscoderEntry("UTF-8", "IBM861", 34460, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0), + new GenericTranscoderEntry("IBM862", "UTF-8", 34984, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0), + new GenericTranscoderEntry("UTF-8", "IBM862", 35228, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0), + new GenericTranscoderEntry("IBM863", "UTF-8", 35752, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0), + new GenericTranscoderEntry("UTF-8", "IBM863", 36132, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0), + new GenericTranscoderEntry("IBM864", "UTF-8", 36644, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0), + new GenericTranscoderEntry("UTF-8", "IBM864", 37504, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0), + new GenericTranscoderEntry("IBM865", "UTF-8", 38028, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0), + new GenericTranscoderEntry("UTF-8", "IBM865", 38340, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0), + new GenericTranscoderEntry("IBM866", "UTF-8", 38864, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0), + new GenericTranscoderEntry("UTF-8", "IBM866", 39320, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0), + new GenericTranscoderEntry("IBM869", "UTF-8", 39812, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0), + new GenericTranscoderEntry("UTF-8", "IBM869", 40328, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0), + new GenericTranscoderEntry("MACCROATIAN", "UTF-8", 40852, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0), + new GenericTranscoderEntry("UTF-8", "MACCROATIAN", 41684, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0), + new GenericTranscoderEntry("MACCYRILLIC", "UTF-8", 42208, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0), + new GenericTranscoderEntry("UTF-8", "MACCYRILLIC", 42912, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0), + new GenericTranscoderEntry("MACGREEK", "UTF-8", 43436, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0), + new GenericTranscoderEntry("UTF-8", "MACGREEK", 44136, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0), + new GenericTranscoderEntry("MACICELAND", "UTF-8", 44660, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0), + new GenericTranscoderEntry("UTF-8", "MACICELAND", 45376, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0), + new GenericTranscoderEntry("MACROMAN", "UTF-8", 45900, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0), + new GenericTranscoderEntry("UTF-8", "MACROMAN", 46384, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0), + new GenericTranscoderEntry("MACROMANIA", "UTF-8", 46908, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0), + new GenericTranscoderEntry("UTF-8", "MACROMANIA", 47284, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0), + new GenericTranscoderEntry("MACTURKISH", "UTF-8", 47804, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0), + new GenericTranscoderEntry("UTF-8", "MACTURKISH", 48160, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0), + new GenericTranscoderEntry("MACUKRAINE", "UTF-8", 48684, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0), + new GenericTranscoderEntry("UTF-8", "MACUKRAINE", 48908, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0), + new GenericTranscoderEntry("KOI8-U", "UTF-8", 49432, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0), + new GenericTranscoderEntry("UTF-8", "KOI8-U", 50216, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0), + new GenericTranscoderEntry("KOI8-R", "UTF-8", 50740, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0), + new GenericTranscoderEntry("UTF-8", "KOI8-R", 51272, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0), + new GenericTranscoderEntry("TIS-620", "UTF-8", 51636, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0), + new GenericTranscoderEntry("UTF-8", "TIS-620", 51680, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0), + new GenericTranscoderEntry("CP850", "UTF-8", 52204, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0), + new GenericTranscoderEntry("UTF-8", "CP850", 52752, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0), + new GenericTranscoderEntry("CP852", "UTF-8", 29992, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0), + new GenericTranscoderEntry("UTF-8", "CP852", 30608, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0), + new GenericTranscoderEntry("CP855", "UTF-8", 31132, "SingleByte", 1, 1, 1, AsciiCompatibility.CONVERTER, 0), + new GenericTranscoderEntry("UTF-8", "CP855", 31684, "SingleByte", 1, 4, 1, AsciiCompatibility.CONVERTER, 0), new GenericTranscoderEntry("UTF-8", "UTF8-MAC", 52420, "Utf8Mac", 1, 4, 9, AsciiCompatibility.CONVERTER, 0) }; public static Transcoder getInstance(String name) { switch (name) { + case "From_CESU_8": return From_CESU_8_Transcoder.INSTANCE; + case "To_CESU_8": return To_CESU_8_Transcoder.INSTANCE; case "Iso2022jp_kddi_decoder": return Iso2022jp_kddi_decoder_Transcoder.INSTANCE; case "Iso2022jp_kddi_encoder": return Iso2022jp_kddi_encoder_Transcoder.INSTANCE; case "Escape_xml_attr_quote": return Escape_xml_attr_quote_Transcoder.INSTANCE; @@ -408,6 +421,7 @@ public static Transcoder getInstance(String name) { case "Eucjp2sjis": return Eucjp2sjis_Transcoder.INSTANCE; case "Sjis2eucjp": return Sjis2eucjp_Transcoder.INSTANCE; case "Universal_newline": return Universal_newline_Transcoder.INSTANCE; + case "Lf_newline": return Lf_newline_Transcoder.INSTANCE; case "From_UTF8_MAC": return From_UTF8_MAC_Transcoder.INSTANCE; case "From_UTF_16BE": return From_UTF_16BE_Transcoder.INSTANCE; case "To_UTF_16BE": return To_UTF_16BE_Transcoder.INSTANCE; diff --git a/src/org/jcodings/transcode/Transcoding.java b/src/org/jcodings/transcode/Transcoding.java index c821ff60..b1e0d8bb 100644 --- a/src/org/jcodings/transcode/Transcoding.java +++ b/src/org/jcodings/transcode/Transcoding.java @@ -21,6 +21,8 @@ import org.jcodings.Ptr; +import static java.lang.Byte.toUnsignedInt; + public class Transcoding implements TranscodingInstruction { public Transcoding(Transcoder transcoder, int flags) { this.transcoder = transcoder; @@ -157,7 +159,7 @@ private EConvResult transcodeRestartable0(final byte[] in_bytes, Ptr in_pos, fin nextByte = in_bytes[in_p++]; // fall through case FOLLOW_BYTE: // follow_byte: - if ((nextByte & 0xFF) < BL_MIN_BYTE(this) || BL_MAX_BYTE(this) < (nextByte & 0xFF)) { + if (toUnsignedInt(nextByte) < BL_MIN_BYTE(this) || BL_MAX_BYTE(this) < toUnsignedInt(nextByte)) { nextInfo = INVALID; } else { nextInfo = BL_ACTION(this, nextByte); @@ -642,19 +644,19 @@ private static int BYTE_LOOKUP_INFO(Transcoding tc, int bl) { } public static int BL_MIN_BYTE(Transcoding tc) { - return tc.transcoder.byteArray[BL_BASE(tc)] & 0xFF; + return toUnsignedInt(tc.transcoder.byteArray[BL_BASE(tc)]); } public static int BL_MAX_BYTE(Transcoding tc) { - return tc.transcoder.byteArray[BL_BASE(tc) + 1] & 0xFF; + return toUnsignedInt(tc.transcoder.byteArray[BL_BASE(tc) + 1]); } public static int BL_OFFSET(Transcoding tc, int b) { - return tc.transcoder.byteArray[BL_BASE(tc) + 2 + b - BL_MIN_BYTE(tc)] & 0xFF; + return toUnsignedInt(tc.transcoder.byteArray[BL_BASE(tc) + 2 + b - BL_MIN_BYTE(tc)]); } public static int BL_ACTION(Transcoding tc, byte b) { - return tc.transcoder.intArray[BL_INFO(tc) + BL_OFFSET(tc, b & 0xFF)]; + return tc.transcoder.intArray[BL_INFO(tc) + BL_OFFSET(tc, toUnsignedInt(b))]; } public static byte getGB4bt0(int a) { @@ -666,7 +668,7 @@ public static byte getGB4bt1(int a) { } public static byte getGB4bt2(int a) { - return (byte)(a >>> 160); + return (byte)(a >>> 16); } public static byte getGB4bt3(int a) { diff --git a/src/org/jcodings/transcode/specific/Cp50220_decoder_Transcoder.java b/src/org/jcodings/transcode/specific/Cp50220_decoder_Transcoder.java index 52060d63..6d0a8124 100644 --- a/src/org/jcodings/transcode/specific/Cp50220_decoder_Transcoder.java +++ b/src/org/jcodings/transcode/specific/Cp50220_decoder_Transcoder.java @@ -40,6 +40,7 @@ public int stateFinish(byte[] state) { return TranscodeFunctions.iso2022jpInit(state); } + @Override public int startToInfo(byte[] statep, byte[] s, int sStart, int l) { return TranscodeFunctions.funSiCp50221Decoder(statep, s, sStart, l); } diff --git a/src/org/jcodings/transcode/specific/Cp50221_decoder_Transcoder.java b/src/org/jcodings/transcode/specific/Cp50221_decoder_Transcoder.java index 78fb56be..b279a244 100644 --- a/src/org/jcodings/transcode/specific/Cp50221_decoder_Transcoder.java +++ b/src/org/jcodings/transcode/specific/Cp50221_decoder_Transcoder.java @@ -40,6 +40,7 @@ public int stateFinish(byte[] state) { return TranscodeFunctions.iso2022jpInit(state); } + @Override public int startToInfo(byte[] statep, byte[] s, int sStart, int l) { return TranscodeFunctions.funSiCp50221Decoder(statep, s, sStart, l); } diff --git a/src/org/jcodings/transcode/specific/Escape_xml_attr_quote_Transcoder.java b/src/org/jcodings/transcode/specific/Escape_xml_attr_quote_Transcoder.java index c4d767f8..c6d5eb24 100644 --- a/src/org/jcodings/transcode/specific/Escape_xml_attr_quote_Transcoder.java +++ b/src/org/jcodings/transcode/specific/Escape_xml_attr_quote_Transcoder.java @@ -25,7 +25,7 @@ public class Escape_xml_attr_quote_Transcoder extends Transcoder { protected Escape_xml_attr_quote_Transcoder () { - super("", "xml_attr_quote", 72, "Escape", 1, 1, 7, AsciiCompatibility.ENCODER, 1); + super("", "xml_attr_quote", 76, "Escape", 1, 1, 7, AsciiCompatibility.ENCODER, 1); } public static final Transcoder INSTANCE = new Escape_xml_attr_quote_Transcoder(); diff --git a/src/org/jcodings/transcode/specific/From_CESU_8_Transcoder.java b/src/org/jcodings/transcode/specific/From_CESU_8_Transcoder.java new file mode 100644 index 00000000..c3cf5e0b --- /dev/null +++ b/src/org/jcodings/transcode/specific/From_CESU_8_Transcoder.java @@ -0,0 +1,37 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.jcodings.transcode.specific; + +import org.jcodings.transcode.AsciiCompatibility; +import org.jcodings.transcode.TranscodeFunctions; +import org.jcodings.transcode.Transcoder; + +public class From_CESU_8_Transcoder extends Transcoder { + protected From_CESU_8_Transcoder () { + super("CESU-8", "UTF-8", 148, "Cesu8", 1, 6, 4, AsciiCompatibility.DECODER, 0); + } + + public static final Transcoder INSTANCE = new From_CESU_8_Transcoder(); + + @Override + public int startToOutput(byte[] statep, byte[] s, int sStart, int l, byte[] o, int oStart, int oSize) { + return TranscodeFunctions.funSoFromCESU8(statep, s, sStart, l, o, oStart, oSize); + } +} diff --git a/src/org/jcodings/transcode/specific/From_UTF8_MAC_Transcoder.java b/src/org/jcodings/transcode/specific/From_UTF8_MAC_Transcoder.java index 92d33556..c6518fd0 100644 --- a/src/org/jcodings/transcode/specific/From_UTF8_MAC_Transcoder.java +++ b/src/org/jcodings/transcode/specific/From_UTF8_MAC_Transcoder.java @@ -25,7 +25,7 @@ public class From_UTF8_MAC_Transcoder extends Transcoder { protected From_UTF8_MAC_Transcoder () { - super("UTF8-MAC", "UTF-8", 4928, "Utf8Mac", 1, 4, 10, AsciiCompatibility.ENCODER, 24); + super("UTF8-MAC", "UTF-8", 52544, "Utf8Mac", 1, 4, 10, AsciiCompatibility.ENCODER, 24); } public static final Transcoder INSTANCE = new From_UTF8_MAC_Transcoder(); diff --git a/src/org/jcodings/transcode/specific/Iso2022jp_decoder_Transcoder.java b/src/org/jcodings/transcode/specific/Iso2022jp_decoder_Transcoder.java index a5fedba9..41a562ea 100644 --- a/src/org/jcodings/transcode/specific/Iso2022jp_decoder_Transcoder.java +++ b/src/org/jcodings/transcode/specific/Iso2022jp_decoder_Transcoder.java @@ -40,6 +40,7 @@ public int stateFinish(byte[] state) { return TranscodeFunctions.iso2022jpInit(state); } + @Override public int startToInfo(byte[] statep, byte[] s, int sStart, int l) { return TranscodeFunctions.funSiIso50220jpDecoder(statep, s, sStart, l); } diff --git a/src/org/jcodings/transcode/specific/Iso2022jp_kddi_decoder_Transcoder.java b/src/org/jcodings/transcode/specific/Iso2022jp_kddi_decoder_Transcoder.java index 57c7eda0..001feb38 100644 --- a/src/org/jcodings/transcode/specific/Iso2022jp_kddi_decoder_Transcoder.java +++ b/src/org/jcodings/transcode/specific/Iso2022jp_kddi_decoder_Transcoder.java @@ -40,6 +40,7 @@ public int stateFinish(byte[] state) { return TranscodeFunctions.iso2022jpKddiInit(state); } + @Override public int startToInfo(byte[] statep, byte[] s, int sStart, int l) { return TranscodeFunctions.funSiIso2022jpKddiDecoder(statep, s, sStart, l); } diff --git a/scripts/UnicodePropertiesTemplate.java b/src/org/jcodings/transcode/specific/Lf_newline_Transcoder.java similarity index 71% rename from scripts/UnicodePropertiesTemplate.java rename to src/org/jcodings/transcode/specific/Lf_newline_Transcoder.java index cd8a253a..94e1e85d 100644 --- a/scripts/UnicodePropertiesTemplate.java +++ b/src/org/jcodings/transcode/specific/Lf_newline_Transcoder.java @@ -17,14 +17,18 @@ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ -package org.jcodings.unicode; +package org.jcodings.transcode.specific; -import org.jcodings.unicode.UnicodeEncoding.CodeRangeEntry; +import org.jcodings.transcode.AsciiCompatibility; +import org.jcodings.transcode.TranscodeFunctions; +import org.jcodings.transcode.Transcoder; +import org.jcodings.transcode.Transcoding; -public class UnicodeProperties { - static final CodeRangeEntry[]CodeRangeTable = new CodeRangeEntry[] { -%{extcrs} - }; +public class Lf_newline_Transcoder extends Universal_newline_Transcoder { - static final int MAX_WORD_LENGTH = %{max_length}; -} + protected Lf_newline_Transcoder() { + super(); + } + + public static final Transcoder INSTANCE = new Lf_newline_Transcoder(); +} \ No newline at end of file diff --git a/src/org/jcodings/transcode/specific/To_CESU_8_Transcoder.java b/src/org/jcodings/transcode/specific/To_CESU_8_Transcoder.java new file mode 100644 index 00000000..cb72c8d7 --- /dev/null +++ b/src/org/jcodings/transcode/specific/To_CESU_8_Transcoder.java @@ -0,0 +1,37 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.jcodings.transcode.specific; + +import org.jcodings.transcode.AsciiCompatibility; +import org.jcodings.transcode.TranscodeFunctions; +import org.jcodings.transcode.Transcoder; + +public class To_CESU_8_Transcoder extends Transcoder { + protected To_CESU_8_Transcoder () { + super("UTF-8", "CESU-8", 240, "Cesu8", 1, 4, 6, AsciiCompatibility.ENCODER, 0); + } + + public static final Transcoder INSTANCE = new To_CESU_8_Transcoder(); + + @Override + public int startToOutput(byte[] statep, byte[] s, int sStart, int l, byte[] o, int oStart, int oSize) { + return TranscodeFunctions.funSoToCESU8(statep, s, sStart, l, o, oStart, oSize); + } +} diff --git a/src/org/jcodings/transcode/specific/Universal_newline_Transcoder.java b/src/org/jcodings/transcode/specific/Universal_newline_Transcoder.java index 3bdd573f..c4a347dc 100644 --- a/src/org/jcodings/transcode/specific/Universal_newline_Transcoder.java +++ b/src/org/jcodings/transcode/specific/Universal_newline_Transcoder.java @@ -26,7 +26,7 @@ public class Universal_newline_Transcoder extends Transcoder { protected Universal_newline_Transcoder() { - super("", "universal_newline", universal_newline, "Newline", 1, 1, 2, AsciiCompatibility.CONVERTER, 2); + super("", "universal_newline", 4, "Newline", 1, 1, 2, AsciiCompatibility.CONVERTER, 2); } private static final int universal_newline = Transcoding.WORDINDEX2INFO(1); diff --git a/src/org/jcodings/unicode/UnicodeCodeRange.java b/src/org/jcodings/unicode/UnicodeCodeRange.java new file mode 100644 index 00000000..111f5478 --- /dev/null +++ b/src/org/jcodings/unicode/UnicodeCodeRange.java @@ -0,0 +1,996 @@ +/* + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + * of the Software, and to permit persons to whom the Software is furnished to do + * so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +package org.jcodings.unicode; + +import org.jcodings.util.ArrayReader; +import org.jcodings.CodeRange; + +public enum UnicodeCodeRange { + NEWLINE("newline", "CR_NEWLINE"), + ALPHA("alpha", "CR_Alpha"), + BLANK("blank", "CR_Blank"), + CNTRL("cntrl", "CR_Cntrl"), + DIGIT("digit", "CR_Digit"), + GRAPH("graph", "CR_Graph"), + LOWER("lower", "CR_Lower"), + PRINT("print", "CR_Print"), + XPOSIXPUNCT("xposixpunct", "CR_XPosixPunct"), + SPACE("space", "CR_Space"), + UPPER("upper", "CR_Upper"), + XDIGIT("xdigit", "CR_XDigit"), + WORD("word", "CR_Word"), + ALNUM("alnum", "CR_Alnum"), + ASCII("ascii", "CR_ASCII"), + PUNCT("punct", "CR_Punct"), + ANY("any", "CR_Any"), + ASSIGNED("assigned", "CR_Assigned"), + C("c", "CR_C"), + OTHER("other", "CR_C"), + CC("cc", "CR_Cntrl"), + CONTROL("control", "CR_Cntrl"), + CF("cf", "CR_Cf"), + FORMAT("format", "CR_Cf"), + CN("cn", "CR_Cn"), + UNASSIGNED("unassigned", "CR_Cn"), + CO("co", "CR_Co"), + PRIVATEUSE("privateuse", "CR_Co"), + CS("cs", "CR_Cs"), + SURROGATE("surrogate", "CR_Cs"), + L("l", "CR_L"), + LETTER("letter", "CR_L"), + LC("lc", "CR_LC"), + CASEDLETTER("casedletter", "CR_LC"), + LL("ll", "CR_Ll"), + LOWERCASELETTER("lowercaseletter", "CR_Ll"), + LM("lm", "CR_Lm"), + MODIFIERLETTER("modifierletter", "CR_Lm"), + LO("lo", "CR_Lo"), + OTHERLETTER("otherletter", "CR_Lo"), + LT("lt", "CR_Lt"), + TITLECASELETTER("titlecaseletter", "CR_Lt"), + LU("lu", "CR_Lu"), + UPPERCASELETTER("uppercaseletter", "CR_Lu"), + M("m", "CR_M"), + MARK("mark", "CR_M"), + COMBININGMARK("combiningmark", "CR_M"), + MC("mc", "CR_Mc"), + SPACINGMARK("spacingmark", "CR_Mc"), + ME("me", "CR_Me"), + ENCLOSINGMARK("enclosingmark", "CR_Me"), + MN("mn", "CR_Mn"), + NONSPACINGMARK("nonspacingmark", "CR_Mn"), + N("n", "CR_N"), + NUMBER("number", "CR_N"), + ND("nd", "CR_Digit"), + DECIMALNUMBER("decimalnumber", "CR_Digit"), + NL("nl", "CR_Nl"), + LETTERNUMBER("letternumber", "CR_Nl"), + NO("no", "CR_No"), + OTHERNUMBER("othernumber", "CR_No"), + P("p", "CR_Punct"), + PUNCTUATION("punctuation", "CR_Punct"), + PC("pc", "CR_Pc"), + CONNECTORPUNCTUATION("connectorpunctuation", "CR_Pc"), + PD("pd", "CR_Pd"), + DASHPUNCTUATION("dashpunctuation", "CR_Pd"), + PE("pe", "CR_Pe"), + CLOSEPUNCTUATION("closepunctuation", "CR_Pe"), + PF("pf", "CR_Pf"), + FINALPUNCTUATION("finalpunctuation", "CR_Pf"), + PI("pi", "CR_Pi"), + INITIALPUNCTUATION("initialpunctuation", "CR_Pi"), + PO("po", "CR_Po"), + OTHERPUNCTUATION("otherpunctuation", "CR_Po"), + PS("ps", "CR_Ps"), + OPENPUNCTUATION("openpunctuation", "CR_Ps"), + S("s", "CR_S"), + SYMBOL("symbol", "CR_S"), + SC("sc", "CR_Sc"), + CURRENCYSYMBOL("currencysymbol", "CR_Sc"), + SK("sk", "CR_Sk"), + MODIFIERSYMBOL("modifiersymbol", "CR_Sk"), + SM("sm", "CR_Sm"), + MATHSYMBOL("mathsymbol", "CR_Sm"), + SO("so", "CR_So"), + OTHERSYMBOL("othersymbol", "CR_So"), + Z("z", "CR_Z"), + SEPARATOR("separator", "CR_Z"), + ZL("zl", "CR_Zl"), + LINESEPARATOR("lineseparator", "CR_Zl"), + ZP("zp", "CR_Zp"), + PARAGRAPHSEPARATOR("paragraphseparator", "CR_Zp"), + ZS("zs", "CR_Zs"), + SPACESEPARATOR("spaceseparator", "CR_Zs"), + MATH("math", "CR_Math"), + ALPHABETIC("alphabetic", "CR_Alpha"), + LOWERCASE("lowercase", "CR_Lower"), + UPPERCASE("uppercase", "CR_Upper"), + CASED("cased", "CR_Cased"), + CASEIGNORABLE("caseignorable", "CR_Case_Ignorable"), + CI("ci", "CR_Case_Ignorable"), + CHANGESWHENLOWERCASED("changeswhenlowercased", "CR_Changes_When_Lowercased"), + CWL("cwl", "CR_Changes_When_Lowercased"), + CHANGESWHENUPPERCASED("changeswhenuppercased", "CR_Changes_When_Uppercased"), + CWU("cwu", "CR_Changes_When_Uppercased"), + CHANGESWHENTITLECASED("changeswhentitlecased", "CR_Changes_When_Titlecased"), + CWT("cwt", "CR_Changes_When_Titlecased"), + CHANGESWHENCASEFOLDED("changeswhencasefolded", "CR_Changes_When_Casefolded"), + CWCF("cwcf", "CR_Changes_When_Casefolded"), + CHANGESWHENCASEMAPPED("changeswhencasemapped", "CR_Changes_When_Casemapped"), + CWCM("cwcm", "CR_Changes_When_Casemapped"), + IDSTART("idstart", "CR_ID_Start"), + IDS("ids", "CR_ID_Start"), + IDCONTINUE("idcontinue", "CR_ID_Continue"), + IDC("idc", "CR_ID_Continue"), + XIDSTART("xidstart", "CR_XID_Start"), + XIDS("xids", "CR_XID_Start"), + XIDCONTINUE("xidcontinue", "CR_XID_Continue"), + XIDC("xidc", "CR_XID_Continue"), + DEFAULTIGNORABLECODEPOINT("defaultignorablecodepoint", "CR_Default_Ignorable_Code_Point"), + DI("di", "CR_Default_Ignorable_Code_Point"), + GRAPHEMEEXTEND("graphemeextend", "CR_Grapheme_Extend"), + GREXT("grext", "CR_Grapheme_Extend"), + GRAPHEMEBASE("graphemebase", "CR_Grapheme_Base"), + GRBASE("grbase", "CR_Grapheme_Base"), + GRAPHEMELINK("graphemelink", "CR_Grapheme_Link"), + GRLINK("grlink", "CR_Grapheme_Link"), + INCBLINKER("incblinker", "CR_InCB_Linker"), + INCBCONSONANT("incbconsonant", "CR_InCB_Consonant"), + INCBEXTEND("incbextend", "CR_InCB_Extend"), + COMMON("common", "CR_Common"), + ZYYY("zyyy", "CR_Common"), + LATIN("latin", "CR_Latin"), + LATN("latn", "CR_Latin"), + GREEK("greek", "CR_Greek"), + GREK("grek", "CR_Greek"), + CYRILLIC("cyrillic", "CR_Cyrillic"), + CYRL("cyrl", "CR_Cyrillic"), + ARMENIAN("armenian", "CR_Armenian"), + ARMN("armn", "CR_Armenian"), + HEBREW("hebrew", "CR_Hebrew"), + HEBR("hebr", "CR_Hebrew"), + ARABIC("arabic", "CR_Arabic"), + ARAB("arab", "CR_Arabic"), + SYRIAC("syriac", "CR_Syriac"), + SYRC("syrc", "CR_Syriac"), + THAANA("thaana", "CR_Thaana"), + THAA("thaa", "CR_Thaana"), + DEVANAGARI("devanagari", "CR_Devanagari"), + DEVA("deva", "CR_Devanagari"), + BENGALI("bengali", "CR_Bengali"), + BENG("beng", "CR_Bengali"), + GURMUKHI("gurmukhi", "CR_Gurmukhi"), + GURU("guru", "CR_Gurmukhi"), + GUJARATI("gujarati", "CR_Gujarati"), + GUJR("gujr", "CR_Gujarati"), + ORIYA("oriya", "CR_Oriya"), + ORYA("orya", "CR_Oriya"), + TAMIL("tamil", "CR_Tamil"), + TAML("taml", "CR_Tamil"), + TELUGU("telugu", "CR_Telugu"), + TELU("telu", "CR_Telugu"), + KANNADA("kannada", "CR_Kannada"), + KNDA("knda", "CR_Kannada"), + MALAYALAM("malayalam", "CR_Malayalam"), + MLYM("mlym", "CR_Malayalam"), + SINHALA("sinhala", "CR_Sinhala"), + SINH("sinh", "CR_Sinhala"), + THAI("thai", "CR_Thai"), + LAO("lao", "CR_Lao"), + LAOO("laoo", "CR_Lao"), + TIBETAN("tibetan", "CR_Tibetan"), + TIBT("tibt", "CR_Tibetan"), + MYANMAR("myanmar", "CR_Myanmar"), + MYMR("mymr", "CR_Myanmar"), + GEORGIAN("georgian", "CR_Georgian"), + GEOR("geor", "CR_Georgian"), + HANGUL("hangul", "CR_Hangul"), + HANG("hang", "CR_Hangul"), + ETHIOPIC("ethiopic", "CR_Ethiopic"), + ETHI("ethi", "CR_Ethiopic"), + CHEROKEE("cherokee", "CR_Cherokee"), + CHER("cher", "CR_Cherokee"), + CANADIANABORIGINAL("canadianaboriginal", "CR_Canadian_Aboriginal"), + CANS("cans", "CR_Canadian_Aboriginal"), + OGHAM("ogham", "CR_Ogham"), + OGAM("ogam", "CR_Ogham"), + RUNIC("runic", "CR_Runic"), + RUNR("runr", "CR_Runic"), + KHMER("khmer", "CR_Khmer"), + KHMR("khmr", "CR_Khmer"), + MONGOLIAN("mongolian", "CR_Mongolian"), + MONG("mong", "CR_Mongolian"), + HIRAGANA("hiragana", "CR_Hiragana"), + HIRA("hira", "CR_Hiragana"), + KATAKANA("katakana", "CR_Katakana"), + KANA("kana", "CR_Katakana"), + BOPOMOFO("bopomofo", "CR_Bopomofo"), + BOPO("bopo", "CR_Bopomofo"), + HAN("han", "CR_Han"), + HANI("hani", "CR_Han"), + YI("yi", "CR_Yi"), + YIII("yiii", "CR_Yi"), + OLDITALIC("olditalic", "CR_Old_Italic"), + ITAL("ital", "CR_Old_Italic"), + GOTHIC("gothic", "CR_Gothic"), + GOTH("goth", "CR_Gothic"), + INDESERET("indeseret", "CR_Deseret"), + INHERITED("inherited", "CR_Inherited"), + QAAI("qaai", "CR_Inherited"), + ZINH("zinh", "CR_Inherited"), + TAGALOG("tagalog", "CR_Tagalog"), + TGLG("tglg", "CR_Tagalog"), + HANUNOO("hanunoo", "CR_Hanunoo"), + HANO("hano", "CR_Hanunoo"), + BUHID("buhid", "CR_Buhid"), + BUHD("buhd", "CR_Buhid"), + TAGBANWA("tagbanwa", "CR_Tagbanwa"), + TAGB("tagb", "CR_Tagbanwa"), + LIMBU("limbu", "CR_Limbu"), + LIMB("limb", "CR_Limbu"), + TAILE("taile", "CR_Tai_Le"), + TALE("tale", "CR_Tai_Le"), + LINEARB("linearb", "CR_Linear_B"), + LINB("linb", "CR_Linear_B"), + UGARITIC("ugaritic", "CR_Ugaritic"), + UGAR("ugar", "CR_Ugaritic"), + INSHAVIAN("inshavian", "CR_Shavian"), + OSMANYA("osmanya", "CR_Osmanya"), + OSMA("osma", "CR_Osmanya"), + CYPRIOT("cypriot", "CR_Cypriot"), + CPRT("cprt", "CR_Cypriot"), + INBRAILLEPATTERNS("inbraillepatterns", "CR_Braille"), + BUGINESE("buginese", "CR_Buginese"), + BUGI("bugi", "CR_Buginese"), + COPTIC("coptic", "CR_Coptic"), + QAAC("qaac", "CR_Coptic"), + COPT("copt", "CR_Coptic"), + NEWTAILUE("newtailue", "CR_New_Tai_Lue"), + TALU("talu", "CR_New_Tai_Lue"), + GLAGOLITIC("glagolitic", "CR_Glagolitic"), + GLAG("glag", "CR_Glagolitic"), + TIFINAGH("tifinagh", "CR_Tifinagh"), + TFNG("tfng", "CR_Tifinagh"), + SYLOTINAGRI("sylotinagri", "CR_Syloti_Nagri"), + SYLO("sylo", "CR_Syloti_Nagri"), + OLDPERSIAN("oldpersian", "CR_Old_Persian"), + XPEO("xpeo", "CR_Old_Persian"), + KHAROSHTHI("kharoshthi", "CR_Kharoshthi"), + KHAR("khar", "CR_Kharoshthi"), + BALINESE("balinese", "CR_Balinese"), + BALI("bali", "CR_Balinese"), + CUNEIFORM("cuneiform", "CR_Cuneiform"), + XSUX("xsux", "CR_Cuneiform"), + PHOENICIAN("phoenician", "CR_Phoenician"), + PHNX("phnx", "CR_Phoenician"), + PHAGSPA("phagspa", "CR_Phags_Pa"), + PHAG("phag", "CR_Phags_Pa"), + NKO("nko", "CR_Nko"), + NKOO("nkoo", "CR_Nko"), + SUNDANESE("sundanese", "CR_Sundanese"), + SUND("sund", "CR_Sundanese"), + LEPCHA("lepcha", "CR_Lepcha"), + LEPC("lepc", "CR_Lepcha"), + INOLCHIKI("inolchiki", "CR_Ol_Chiki"), + VAI("vai", "CR_Vai"), + VAII("vaii", "CR_Vai"), + SAURASHTRA("saurashtra", "CR_Saurashtra"), + SAUR("saur", "CR_Saurashtra"), + KAYAHLI("kayahli", "CR_Kayah_Li"), + KALI("kali", "CR_Kayah_Li"), + REJANG("rejang", "CR_Rejang"), + RJNG("rjng", "CR_Rejang"), + LYCIAN("lycian", "CR_Lycian"), + LYCI("lyci", "CR_Lycian"), + CARIAN("carian", "CR_Carian"), + CARI("cari", "CR_Carian"), + LYDIAN("lydian", "CR_Lydian"), + LYDI("lydi", "CR_Lydian"), + CHAM("cham", "CR_Cham"), + TAITHAM("taitham", "CR_Tai_Tham"), + LANA("lana", "CR_Tai_Tham"), + TAIVIET("taiviet", "CR_Tai_Viet"), + TAVT("tavt", "CR_Tai_Viet"), + AVESTAN("avestan", "CR_Avestan"), + AVST("avst", "CR_Avestan"), + EGYPTIANHIEROGLYPHS("egyptianhieroglyphs", "CR_Egyptian_Hieroglyphs"), + EGYP("egyp", "CR_Egyptian_Hieroglyphs"), + SAMARITAN("samaritan", "CR_Samaritan"), + SAMR("samr", "CR_Samaritan"), + LISU("lisu", "CR_Lisu"), + BAMUM("bamum", "CR_Bamum"), + BAMU("bamu", "CR_Bamum"), + JAVANESE("javanese", "CR_Javanese"), + JAVA("java", "CR_Javanese"), + MEETEIMAYEK("meeteimayek", "CR_Meetei_Mayek"), + MTEI("mtei", "CR_Meetei_Mayek"), + IMPERIALARAMAIC("imperialaramaic", "CR_Imperial_Aramaic"), + ARMI("armi", "CR_Imperial_Aramaic"), + INOLDSOUTHARABIAN("inoldsoutharabian", "CR_Old_South_Arabian"), + INSCRIPTIONALPARTHIAN("inscriptionalparthian", "CR_Inscriptional_Parthian"), + PRTI("prti", "CR_Inscriptional_Parthian"), + INSCRIPTIONALPAHLAVI("inscriptionalpahlavi", "CR_Inscriptional_Pahlavi"), + PHLI("phli", "CR_Inscriptional_Pahlavi"), + OLDTURKIC("oldturkic", "CR_Old_Turkic"), + ORKH("orkh", "CR_Old_Turkic"), + KAITHI("kaithi", "CR_Kaithi"), + KTHI("kthi", "CR_Kaithi"), + BATAK("batak", "CR_Batak"), + BATK("batk", "CR_Batak"), + BRAHMI("brahmi", "CR_Brahmi"), + BRAH("brah", "CR_Brahmi"), + MANDAIC("mandaic", "CR_Mandaic"), + MAND("mand", "CR_Mandaic"), + CHAKMA("chakma", "CR_Chakma"), + CAKM("cakm", "CR_Chakma"), + MEROITICCURSIVE("meroiticcursive", "CR_Meroitic_Cursive"), + MERC("merc", "CR_Meroitic_Cursive"), + INMEROITICHIEROGLYPHS("inmeroitichieroglyphs", "CR_Meroitic_Hieroglyphs"), + MIAO("miao", "CR_Miao"), + PLRD("plrd", "CR_Miao"), + SHARADA("sharada", "CR_Sharada"), + SHRD("shrd", "CR_Sharada"), + SORASOMPENG("sorasompeng", "CR_Sora_Sompeng"), + SORA("sora", "CR_Sora_Sompeng"), + TAKRI("takri", "CR_Takri"), + TAKR("takr", "CR_Takri"), + CAUCASIANALBANIAN("caucasianalbanian", "CR_Caucasian_Albanian"), + AGHB("aghb", "CR_Caucasian_Albanian"), + BASSAVAH("bassavah", "CR_Bassa_Vah"), + BASS("bass", "CR_Bassa_Vah"), + DUPLOYAN("duployan", "CR_Duployan"), + DUPL("dupl", "CR_Duployan"), + ELBASAN("elbasan", "CR_Elbasan"), + ELBA("elba", "CR_Elbasan"), + GRANTHA("grantha", "CR_Grantha"), + GRAN("gran", "CR_Grantha"), + PAHAWHHMONG("pahawhhmong", "CR_Pahawh_Hmong"), + HMNG("hmng", "CR_Pahawh_Hmong"), + KHOJKI("khojki", "CR_Khojki"), + KHOJ("khoj", "CR_Khojki"), + LINEARA("lineara", "CR_Linear_A"), + LINA("lina", "CR_Linear_A"), + MAHAJANI("mahajani", "CR_Mahajani"), + MAHJ("mahj", "CR_Mahajani"), + MANICHAEAN("manichaean", "CR_Manichaean"), + MANI("mani", "CR_Manichaean"), + MENDEKIKAKUI("mendekikakui", "CR_Mende_Kikakui"), + MEND("mend", "CR_Mende_Kikakui"), + MODI("modi", "CR_Modi"), + MRO("mro", "CR_Mro"), + MROO("mroo", "CR_Mro"), + INOLDNORTHARABIAN("inoldnortharabian", "CR_Old_North_Arabian"), + NABATAEAN("nabataean", "CR_Nabataean"), + NBAT("nbat", "CR_Nabataean"), + INPALMYRENE("inpalmyrene", "CR_Palmyrene"), + PAUCINHAU("paucinhau", "CR_Pau_Cin_Hau"), + PAUC("pauc", "CR_Pau_Cin_Hau"), + OLDPERMIC("oldpermic", "CR_Old_Permic"), + PERM("perm", "CR_Old_Permic"), + PSALTERPAHLAVI("psalterpahlavi", "CR_Psalter_Pahlavi"), + PHLP("phlp", "CR_Psalter_Pahlavi"), + SIDDHAM("siddham", "CR_Siddham"), + SIDD("sidd", "CR_Siddham"), + KHUDAWADI("khudawadi", "CR_Khudawadi"), + SIND("sind", "CR_Khudawadi"), + TIRHUTA("tirhuta", "CR_Tirhuta"), + TIRH("tirh", "CR_Tirhuta"), + WARANGCITI("warangciti", "CR_Warang_Citi"), + WARA("wara", "CR_Warang_Citi"), + AHOM("ahom", "CR_Ahom"), + ANATOLIANHIEROGLYPHS("anatolianhieroglyphs", "CR_Anatolian_Hieroglyphs"), + HLUW("hluw", "CR_Anatolian_Hieroglyphs"), + HATRAN("hatran", "CR_Hatran"), + HATR("hatr", "CR_Hatran"), + MULTANI("multani", "CR_Multani"), + MULT("mult", "CR_Multani"), + OLDHUNGARIAN("oldhungarian", "CR_Old_Hungarian"), + HUNG("hung", "CR_Old_Hungarian"), + SIGNWRITING("signwriting", "CR_SignWriting"), + SGNW("sgnw", "CR_SignWriting"), + ADLAM("adlam", "CR_Adlam"), + ADLM("adlm", "CR_Adlam"), + BHAIKSUKI("bhaiksuki", "CR_Bhaiksuki"), + BHKS("bhks", "CR_Bhaiksuki"), + MARCHEN("marchen", "CR_Marchen"), + MARC("marc", "CR_Marchen"), + NEWA("newa", "CR_Newa"), + OSAGE("osage", "CR_Osage"), + OSGE("osge", "CR_Osage"), + TANGUT("tangut", "CR_Tangut"), + TANG("tang", "CR_Tangut"), + MASARAMGONDI("masaramgondi", "CR_Masaram_Gondi"), + GONM("gonm", "CR_Masaram_Gondi"), + NUSHU("nushu", "CR_Nushu"), + NSHU("nshu", "CR_Nushu"), + SOYOMBO("soyombo", "CR_Soyombo"), + SOYO("soyo", "CR_Soyombo"), + ZANABAZARSQUARE("zanabazarsquare", "CR_Zanabazar_Square"), + ZANB("zanb", "CR_Zanabazar_Square"), + DOGRA("dogra", "CR_Dogra"), + DOGR("dogr", "CR_Dogra"), + GUNJALAGONDI("gunjalagondi", "CR_Gunjala_Gondi"), + GONG("gong", "CR_Gunjala_Gondi"), + MAKASAR("makasar", "CR_Makasar"), + MAKA("maka", "CR_Makasar"), + MEDEFAIDRIN("medefaidrin", "CR_Medefaidrin"), + MEDF("medf", "CR_Medefaidrin"), + HANIFIROHINGYA("hanifirohingya", "CR_Hanifi_Rohingya"), + ROHG("rohg", "CR_Hanifi_Rohingya"), + SOGDIAN("sogdian", "CR_Sogdian"), + SOGD("sogd", "CR_Sogdian"), + OLDSOGDIAN("oldsogdian", "CR_Old_Sogdian"), + SOGO("sogo", "CR_Old_Sogdian"), + ELYMAIC("elymaic", "CR_Elymaic"), + ELYM("elym", "CR_Elymaic"), + NANDINAGARI("nandinagari", "CR_Nandinagari"), + NAND("nand", "CR_Nandinagari"), + NYIAKENGPUACHUEHMONG("nyiakengpuachuehmong", "CR_Nyiakeng_Puachue_Hmong"), + HMNP("hmnp", "CR_Nyiakeng_Puachue_Hmong"), + WANCHO("wancho", "CR_Wancho"), + WCHO("wcho", "CR_Wancho"), + CHORASMIAN("chorasmian", "CR_Chorasmian"), + CHRS("chrs", "CR_Chorasmian"), + DIVESAKURU("divesakuru", "CR_Dives_Akuru"), + DIAK("diak", "CR_Dives_Akuru"), + KHITANSMALLSCRIPT("khitansmallscript", "CR_Khitan_Small_Script"), + KITS("kits", "CR_Khitan_Small_Script"), + YEZIDI("yezidi", "CR_Yezidi"), + YEZI("yezi", "CR_Yezidi"), + CYPROMINOAN("cyprominoan", "CR_Cypro_Minoan"), + CPMN("cpmn", "CR_Cypro_Minoan"), + OLDUYGHUR("olduyghur", "CR_Old_Uyghur"), + OUGR("ougr", "CR_Old_Uyghur"), + TANGSA("tangsa", "CR_Tangsa"), + TNSA("tnsa", "CR_Tangsa"), + TOTO("toto", "CR_Toto"), + VITHKUQI("vithkuqi", "CR_Vithkuqi"), + VITH("vith", "CR_Vithkuqi"), + KAWI("kawi", "CR_Kawi"), + NAGMUNDARI("nagmundari", "CR_Nag_Mundari"), + NAGM("nagm", "CR_Nag_Mundari"), + GARAY("garay", "CR_Garay"), + GARA("gara", "CR_Garay"), + GURUNGKHEMA("gurungkhema", "CR_Gurung_Khema"), + GUKH("gukh", "CR_Gurung_Khema"), + KIRATRAI("kiratrai", "CR_Kirat_Rai"), + KRAI("krai", "CR_Kirat_Rai"), + OLONAL("olonal", "CR_Ol_Onal"), + ONAO("onao", "CR_Ol_Onal"), + SUNUWAR("sunuwar", "CR_Sunuwar"), + SUNU("sunu", "CR_Sunuwar"), + TODHRI("todhri", "CR_Todhri"), + TODR("todr", "CR_Todhri"), + TULUTIGALARI("tulutigalari", "CR_Tulu_Tigalari"), + TUTG("tutg", "CR_Tulu_Tigalari"), + SIDETIC("sidetic", "CR_Sidetic"), + SIDT("sidt", "CR_Sidetic"), + TAIYO("taiyo", "CR_Tai_Yo"), + TAYO("tayo", "CR_Tai_Yo"), + TOLONGSIKI("tolongsiki", "CR_Tolong_Siki"), + TOLS("tols", "CR_Tolong_Siki"), + BERIAERFE("beriaerfe", "CR_Beria_Erfe"), + BERF("berf", "CR_Beria_Erfe"), + WHITESPACE("whitespace", "CR_Space"), + WSPACE("wspace", "CR_Space"), + BIDICONTROL("bidicontrol", "CR_Bidi_Control"), + BIDIC("bidic", "CR_Bidi_Control"), + JOINCONTROL("joincontrol", "CR_Join_Control"), + JOINC("joinc", "CR_Join_Control"), + DASH("dash", "CR_Dash"), + HYPHEN("hyphen", "CR_Hyphen"), + QUOTATIONMARK("quotationmark", "CR_Quotation_Mark"), + QMARK("qmark", "CR_Quotation_Mark"), + TERMINALPUNCTUATION("terminalpunctuation", "CR_Terminal_Punctuation"), + TERM("term", "CR_Terminal_Punctuation"), + OTHERMATH("othermath", "CR_Other_Math"), + OMATH("omath", "CR_Other_Math"), + HEXDIGIT("hexdigit", "CR_Hex_Digit"), + HEX("hex", "CR_Hex_Digit"), + ASCIIHEXDIGIT("asciihexdigit", "CR_XDigit"), + AHEX("ahex", "CR_XDigit"), + OTHERALPHABETIC("otheralphabetic", "CR_Other_Alphabetic"), + OALPHA("oalpha", "CR_Other_Alphabetic"), + IDEOGRAPHIC("ideographic", "CR_Ideographic"), + IDEO("ideo", "CR_Ideographic"), + DIACRITIC("diacritic", "CR_Diacritic"), + DIA("dia", "CR_Diacritic"), + EXTENDER("extender", "CR_Extender"), + EXT("ext", "CR_Extender"), + OTHERLOWERCASE("otherlowercase", "CR_Other_Lowercase"), + OLOWER("olower", "CR_Other_Lowercase"), + OTHERUPPERCASE("otheruppercase", "CR_Other_Uppercase"), + OUPPER("oupper", "CR_Other_Uppercase"), + NONCHARACTERCODEPOINT("noncharactercodepoint", "CR_Noncharacter_Code_Point"), + NCHAR("nchar", "CR_Noncharacter_Code_Point"), + OTHERGRAPHEMEEXTEND("othergraphemeextend", "CR_Other_Grapheme_Extend"), + OGREXT("ogrext", "CR_Other_Grapheme_Extend"), + IDSBINARYOPERATOR("idsbinaryoperator", "CR_IDS_Binary_Operator"), + IDSB("idsb", "CR_IDS_Binary_Operator"), + IDSTRINARYOPERATOR("idstrinaryoperator", "CR_IDS_Trinary_Operator"), + IDST("idst", "CR_IDS_Trinary_Operator"), + IDSUNARYOPERATOR("idsunaryoperator", "CR_IDS_Unary_Operator"), + IDSU("idsu", "CR_IDS_Unary_Operator"), + RADICAL("radical", "CR_Radical"), + UNIFIEDIDEOGRAPH("unifiedideograph", "CR_Unified_Ideograph"), + UIDEO("uideo", "CR_Unified_Ideograph"), + OTHERDEFAULTIGNORABLECODEPOINT("otherdefaultignorablecodepoint", "CR_Other_Default_Ignorable_Code_Point"), + ODI("odi", "CR_Other_Default_Ignorable_Code_Point"), + DEPRECATED("deprecated", "CR_Deprecated"), + DEP("dep", "CR_Deprecated"), + SOFTDOTTED("softdotted", "CR_Soft_Dotted"), + SD("sd", "CR_Soft_Dotted"), + LOGICALORDEREXCEPTION("logicalorderexception", "CR_Logical_Order_Exception"), + LOE("loe", "CR_Logical_Order_Exception"), + OTHERIDSTART("otheridstart", "CR_Other_ID_Start"), + OIDS("oids", "CR_Other_ID_Start"), + OTHERIDCONTINUE("otheridcontinue", "CR_Other_ID_Continue"), + OIDC("oidc", "CR_Other_ID_Continue"), + IDCOMPATMATHCONTINUE("idcompatmathcontinue", "CR_ID_Compat_Math_Continue"), + IDCOMPATMATHSTART("idcompatmathstart", "CR_ID_Compat_Math_Start"), + SENTENCETERMINAL("sentenceterminal", "CR_Sentence_Terminal"), + STERM("sterm", "CR_Sentence_Terminal"), + VARIATIONSELECTOR("variationselector", "CR_Variation_Selector"), + VS("vs", "CR_Variation_Selector"), + PATTERNWHITESPACE("patternwhitespace", "CR_Pattern_White_Space"), + PATWS("patws", "CR_Pattern_White_Space"), + PATTERNSYNTAX("patternsyntax", "CR_Pattern_Syntax"), + PATSYN("patsyn", "CR_Pattern_Syntax"), + PREPENDEDCONCATENATIONMARK("prependedconcatenationmark", "CR_Prepended_Concatenation_Mark"), + PCM("pcm", "CR_Prepended_Concatenation_Mark"), + GRAPHEMECLUSTERBREAK_REGIONALINDICATOR("graphemeclusterbreak=regionalindicator", "CR_Regional_Indicator"), + MODIFIERCOMBININGMARK("modifiercombiningmark", "CR_Modifier_Combining_Mark"), + MCM("mcm", "CR_Modifier_Combining_Mark"), + EMOJI("emoji", "CR_Emoji"), + EMOJIPRESENTATION("emojipresentation", "CR_Emoji_Presentation"), + EPRES("epres", "CR_Emoji_Presentation"), + EMOJIMODIFIER("emojimodifier", "CR_Emoji_Modifier"), + EMOD("emod", "CR_Emoji_Modifier"), + EMOJIMODIFIERBASE("emojimodifierbase", "CR_Emoji_Modifier_Base"), + EBASE("ebase", "CR_Emoji_Modifier_Base"), + EMOJICOMPONENT("emojicomponent", "CR_Emoji_Component"), + ECOMP("ecomp", "CR_Emoji_Component"), + EXTENDEDPICTOGRAPHIC("extendedpictographic", "CR_Extended_Pictographic"), + EXTPICT("extpict", "CR_Extended_Pictographic"), + UNKNOWN("unknown", "CR_Unknown"), + ZZZZ("zzzz", "CR_Unknown"), + AGE_1_1("age=1.1", "CR_Age_1_1"), + AGE_2_0("age=2.0", "CR_Age_2_0"), + AGE_2_1("age=2.1", "CR_Age_2_1"), + AGE_3_0("age=3.0", "CR_Age_3_0"), + AGE_3_1("age=3.1", "CR_Age_3_1"), + AGE_3_2("age=3.2", "CR_Age_3_2"), + AGE_4_0("age=4.0", "CR_Age_4_0"), + AGE_4_1("age=4.1", "CR_Age_4_1"), + AGE_5_0("age=5.0", "CR_Age_5_0"), + AGE_5_1("age=5.1", "CR_Age_5_1"), + AGE_5_2("age=5.2", "CR_Age_5_2"), + AGE_6_0("age=6.0", "CR_Age_6_0"), + AGE_6_1("age=6.1", "CR_Age_6_1"), + AGE_6_2("age=6.2", "CR_Age_6_2"), + AGE_6_3("age=6.3", "CR_Age_6_3"), + AGE_7_0("age=7.0", "CR_Age_7_0"), + AGE_8_0("age=8.0", "CR_Age_8_0"), + AGE_9_0("age=9.0", "CR_Age_9_0"), + AGE_10_0("age=10.0", "CR_Age_10_0"), + AGE_11_0("age=11.0", "CR_Age_11_0"), + AGE_12_0("age=12.0", "CR_Age_12_0"), + AGE_12_1("age=12.1", "CR_Age_12_1"), + AGE_13_0("age=13.0", "CR_Age_13_0"), + AGE_14_0("age=14.0", "CR_Age_14_0"), + AGE_15_0("age=15.0", "CR_Age_15_0"), + AGE_15_1("age=15.1", "CR_Age_15_1"), + AGE_16_0("age=16.0", "CR_Age_16_0"), + AGE_17_0("age=17.0", "CR_Age_17_0"), + GRAPHEMECLUSTERBREAK_PREPEND("graphemeclusterbreak=prepend", "CR_Grapheme_Cluster_Break_Prepend"), + GRAPHEMECLUSTERBREAK_CR("graphemeclusterbreak=cr", "CR_Grapheme_Cluster_Break_CR"), + GRAPHEMECLUSTERBREAK_LF("graphemeclusterbreak=lf", "CR_NEWLINE"), + GRAPHEMECLUSTERBREAK_CONTROL("graphemeclusterbreak=control", "CR_Grapheme_Cluster_Break_Control"), + GRAPHEMECLUSTERBREAK_EXTEND("graphemeclusterbreak=extend", "CR_Grapheme_Cluster_Break_Extend"), + REGIONALINDICATOR("regionalindicator", "CR_Regional_Indicator"), + RI("ri", "CR_Regional_Indicator"), + GRAPHEMECLUSTERBREAK_SPACINGMARK("graphemeclusterbreak=spacingmark", "CR_Grapheme_Cluster_Break_SpacingMark"), + GRAPHEMECLUSTERBREAK_L("graphemeclusterbreak=l", "CR_Grapheme_Cluster_Break_L"), + GRAPHEMECLUSTERBREAK_V("graphemeclusterbreak=v", "CR_Grapheme_Cluster_Break_V"), + GRAPHEMECLUSTERBREAK_T("graphemeclusterbreak=t", "CR_Grapheme_Cluster_Break_T"), + GRAPHEMECLUSTERBREAK_LV("graphemeclusterbreak=lv", "CR_Grapheme_Cluster_Break_LV"), + GRAPHEMECLUSTERBREAK_LVT("graphemeclusterbreak=lvt", "CR_Grapheme_Cluster_Break_LVT"), + GRAPHEMECLUSTERBREAK_ZWJ("graphemeclusterbreak=zwj", "CR_Grapheme_Cluster_Break_ZWJ"), + INBASICLATIN("inbasiclatin", "CR_ASCII"), + INLATIN1SUPPLEMENT("inlatin1supplement", "CR_In_Latin_1_Supplement"), + INLATINEXTENDEDA("inlatinextendeda", "CR_In_Latin_Extended_A"), + INLATINEXTENDEDB("inlatinextendedb", "CR_In_Latin_Extended_B"), + INIPAEXTENSIONS("inipaextensions", "CR_In_IPA_Extensions"), + INSPACINGMODIFIERLETTERS("inspacingmodifierletters", "CR_In_Spacing_Modifier_Letters"), + INCOMBININGDIACRITICALMARKS("incombiningdiacriticalmarks", "CR_In_Combining_Diacritical_Marks"), + INGREEKANDCOPTIC("ingreekandcoptic", "CR_In_Greek_and_Coptic"), + INCYRILLIC("incyrillic", "CR_In_Cyrillic"), + INCYRILLICSUPPLEMENT("incyrillicsupplement", "CR_In_Cyrillic_Supplement"), + INARMENIAN("inarmenian", "CR_In_Armenian"), + INHEBREW("inhebrew", "CR_In_Hebrew"), + INARABIC("inarabic", "CR_In_Arabic"), + INSYRIAC("insyriac", "CR_In_Syriac"), + INARABICSUPPLEMENT("inarabicsupplement", "CR_In_Arabic_Supplement"), + INTHAANA("inthaana", "CR_In_Thaana"), + INNKO("innko", "CR_In_NKo"), + INSAMARITAN("insamaritan", "CR_In_Samaritan"), + INMANDAIC("inmandaic", "CR_In_Mandaic"), + INSYRIACSUPPLEMENT("insyriacsupplement", "CR_In_Syriac_Supplement"), + INARABICEXTENDEDB("inarabicextendedb", "CR_In_Arabic_Extended_B"), + INARABICEXTENDEDA("inarabicextendeda", "CR_In_Arabic_Extended_A"), + INDEVANAGARI("indevanagari", "CR_In_Devanagari"), + INBENGALI("inbengali", "CR_In_Bengali"), + INGURMUKHI("ingurmukhi", "CR_In_Gurmukhi"), + INGUJARATI("ingujarati", "CR_In_Gujarati"), + INORIYA("inoriya", "CR_In_Oriya"), + INTAMIL("intamil", "CR_In_Tamil"), + INTELUGU("intelugu", "CR_In_Telugu"), + INKANNADA("inkannada", "CR_In_Kannada"), + INMALAYALAM("inmalayalam", "CR_In_Malayalam"), + INSINHALA("insinhala", "CR_In_Sinhala"), + INTHAI("inthai", "CR_In_Thai"), + INLAO("inlao", "CR_In_Lao"), + INTIBETAN("intibetan", "CR_In_Tibetan"), + INMYANMAR("inmyanmar", "CR_In_Myanmar"), + INGEORGIAN("ingeorgian", "CR_In_Georgian"), + INHANGULJAMO("inhanguljamo", "CR_In_Hangul_Jamo"), + INETHIOPIC("inethiopic", "CR_In_Ethiopic"), + INETHIOPICSUPPLEMENT("inethiopicsupplement", "CR_In_Ethiopic_Supplement"), + INCHEROKEE("incherokee", "CR_In_Cherokee"), + INUNIFIEDCANADIANABORIGINALSYLLABICS("inunifiedcanadianaboriginalsyllabics", "CR_In_Unified_Canadian_Aboriginal_Syllabics"), + INOGHAM("inogham", "CR_In_Ogham"), + INRUNIC("inrunic", "CR_In_Runic"), + INTAGALOG("intagalog", "CR_In_Tagalog"), + INHANUNOO("inhanunoo", "CR_In_Hanunoo"), + INBUHID("inbuhid", "CR_In_Buhid"), + INTAGBANWA("intagbanwa", "CR_In_Tagbanwa"), + INKHMER("inkhmer", "CR_In_Khmer"), + INMONGOLIAN("inmongolian", "CR_In_Mongolian"), + INUNIFIEDCANADIANABORIGINALSYLLABICSEXTENDED("inunifiedcanadianaboriginalsyllabicsextended", "CR_In_Unified_Canadian_Aboriginal_Syllabics_Extended"), + INLIMBU("inlimbu", "CR_In_Limbu"), + INTAILE("intaile", "CR_In_Tai_Le"), + INNEWTAILUE("innewtailue", "CR_In_New_Tai_Lue"), + INKHMERSYMBOLS("inkhmersymbols", "CR_In_Khmer_Symbols"), + INBUGINESE("inbuginese", "CR_In_Buginese"), + INTAITHAM("intaitham", "CR_In_Tai_Tham"), + INCOMBININGDIACRITICALMARKSEXTENDED("incombiningdiacriticalmarksextended", "CR_In_Combining_Diacritical_Marks_Extended"), + INBALINESE("inbalinese", "CR_In_Balinese"), + INSUNDANESE("insundanese", "CR_In_Sundanese"), + INBATAK("inbatak", "CR_In_Batak"), + INLEPCHA("inlepcha", "CR_In_Lepcha"), + OLCHIKI("olchiki", "CR_Ol_Chiki"), + OLCK("olck", "CR_Ol_Chiki"), + INCYRILLICEXTENDEDC("incyrillicextendedc", "CR_In_Cyrillic_Extended_C"), + INGEORGIANEXTENDED("ingeorgianextended", "CR_In_Georgian_Extended"), + INSUNDANESESUPPLEMENT("insundanesesupplement", "CR_In_Sundanese_Supplement"), + INVEDICEXTENSIONS("invedicextensions", "CR_In_Vedic_Extensions"), + INPHONETICEXTENSIONS("inphoneticextensions", "CR_In_Phonetic_Extensions"), + INPHONETICEXTENSIONSSUPPLEMENT("inphoneticextensionssupplement", "CR_In_Phonetic_Extensions_Supplement"), + INCOMBININGDIACRITICALMARKSSUPPLEMENT("incombiningdiacriticalmarkssupplement", "CR_In_Combining_Diacritical_Marks_Supplement"), + INLATINEXTENDEDADDITIONAL("inlatinextendedadditional", "CR_In_Latin_Extended_Additional"), + INGREEKEXTENDED("ingreekextended", "CR_In_Greek_Extended"), + INGENERALPUNCTUATION("ingeneralpunctuation", "CR_In_General_Punctuation"), + INSUPERSCRIPTSANDSUBSCRIPTS("insuperscriptsandsubscripts", "CR_In_Superscripts_and_Subscripts"), + INCURRENCYSYMBOLS("incurrencysymbols", "CR_In_Currency_Symbols"), + INCOMBININGDIACRITICALMARKSFORSYMBOLS("incombiningdiacriticalmarksforsymbols", "CR_In_Combining_Diacritical_Marks_for_Symbols"), + INLETTERLIKESYMBOLS("inletterlikesymbols", "CR_In_Letterlike_Symbols"), + INNUMBERFORMS("innumberforms", "CR_In_Number_Forms"), + INARROWS("inarrows", "CR_In_Arrows"), + INMATHEMATICALOPERATORS("inmathematicaloperators", "CR_In_Mathematical_Operators"), + INMISCELLANEOUSTECHNICAL("inmiscellaneoustechnical", "CR_In_Miscellaneous_Technical"), + INCONTROLPICTURES("incontrolpictures", "CR_In_Control_Pictures"), + INOPTICALCHARACTERRECOGNITION("inopticalcharacterrecognition", "CR_In_Optical_Character_Recognition"), + INENCLOSEDALPHANUMERICS("inenclosedalphanumerics", "CR_In_Enclosed_Alphanumerics"), + INBOXDRAWING("inboxdrawing", "CR_In_Box_Drawing"), + INBLOCKELEMENTS("inblockelements", "CR_In_Block_Elements"), + INGEOMETRICSHAPES("ingeometricshapes", "CR_In_Geometric_Shapes"), + INMISCELLANEOUSSYMBOLS("inmiscellaneoussymbols", "CR_In_Miscellaneous_Symbols"), + INDINGBATS("indingbats", "CR_In_Dingbats"), + INMISCELLANEOUSMATHEMATICALSYMBOLSA("inmiscellaneousmathematicalsymbolsa", "CR_In_Miscellaneous_Mathematical_Symbols_A"), + INSUPPLEMENTALARROWSA("insupplementalarrowsa", "CR_In_Supplemental_Arrows_A"), + BRAILLE("braille", "CR_Braille"), + BRAI("brai", "CR_Braille"), + INSUPPLEMENTALARROWSB("insupplementalarrowsb", "CR_In_Supplemental_Arrows_B"), + INMISCELLANEOUSMATHEMATICALSYMBOLSB("inmiscellaneousmathematicalsymbolsb", "CR_In_Miscellaneous_Mathematical_Symbols_B"), + INSUPPLEMENTALMATHEMATICALOPERATORS("insupplementalmathematicaloperators", "CR_In_Supplemental_Mathematical_Operators"), + INMISCELLANEOUSSYMBOLSANDARROWS("inmiscellaneoussymbolsandarrows", "CR_In_Miscellaneous_Symbols_and_Arrows"), + INGLAGOLITIC("inglagolitic", "CR_In_Glagolitic"), + INLATINEXTENDEDC("inlatinextendedc", "CR_In_Latin_Extended_C"), + INCOPTIC("incoptic", "CR_In_Coptic"), + INGEORGIANSUPPLEMENT("ingeorgiansupplement", "CR_In_Georgian_Supplement"), + INTIFINAGH("intifinagh", "CR_In_Tifinagh"), + INETHIOPICEXTENDED("inethiopicextended", "CR_In_Ethiopic_Extended"), + INCYRILLICEXTENDEDA("incyrillicextendeda", "CR_In_Cyrillic_Extended_A"), + INSUPPLEMENTALPUNCTUATION("insupplementalpunctuation", "CR_In_Supplemental_Punctuation"), + INCJKRADICALSSUPPLEMENT("incjkradicalssupplement", "CR_In_CJK_Radicals_Supplement"), + INKANGXIRADICALS("inkangxiradicals", "CR_In_Kangxi_Radicals"), + INIDEOGRAPHICDESCRIPTIONCHARACTERS("inideographicdescriptioncharacters", "CR_In_Ideographic_Description_Characters"), + INCJKSYMBOLSANDPUNCTUATION("incjksymbolsandpunctuation", "CR_In_CJK_Symbols_and_Punctuation"), + INHIRAGANA("inhiragana", "CR_In_Hiragana"), + INKATAKANA("inkatakana", "CR_In_Katakana"), + INBOPOMOFO("inbopomofo", "CR_In_Bopomofo"), + INHANGULCOMPATIBILITYJAMO("inhangulcompatibilityjamo", "CR_In_Hangul_Compatibility_Jamo"), + INKANBUN("inkanbun", "CR_In_Kanbun"), + INBOPOMOFOEXTENDED("inbopomofoextended", "CR_In_Bopomofo_Extended"), + INCJKSTROKES("incjkstrokes", "CR_In_CJK_Strokes"), + INKATAKANAPHONETICEXTENSIONS("inkatakanaphoneticextensions", "CR_In_Katakana_Phonetic_Extensions"), + INENCLOSEDCJKLETTERSANDMONTHS("inenclosedcjklettersandmonths", "CR_In_Enclosed_CJK_Letters_and_Months"), + INCJKCOMPATIBILITY("incjkcompatibility", "CR_In_CJK_Compatibility"), + INCJKUNIFIEDIDEOGRAPHSEXTENSIONA("incjkunifiedideographsextensiona", "CR_In_CJK_Unified_Ideographs_Extension_A"), + INYIJINGHEXAGRAMSYMBOLS("inyijinghexagramsymbols", "CR_In_Yijing_Hexagram_Symbols"), + INCJKUNIFIEDIDEOGRAPHS("incjkunifiedideographs", "CR_In_CJK_Unified_Ideographs"), + INYISYLLABLES("inyisyllables", "CR_In_Yi_Syllables"), + INYIRADICALS("inyiradicals", "CR_In_Yi_Radicals"), + INLISU("inlisu", "CR_In_Lisu"), + INVAI("invai", "CR_In_Vai"), + INCYRILLICEXTENDEDB("incyrillicextendedb", "CR_In_Cyrillic_Extended_B"), + INBAMUM("inbamum", "CR_In_Bamum"), + INMODIFIERTONELETTERS("inmodifiertoneletters", "CR_In_Modifier_Tone_Letters"), + INLATINEXTENDEDD("inlatinextendedd", "CR_In_Latin_Extended_D"), + INSYLOTINAGRI("insylotinagri", "CR_In_Syloti_Nagri"), + INCOMMONINDICNUMBERFORMS("incommonindicnumberforms", "CR_In_Common_Indic_Number_Forms"), + INPHAGSPA("inphagspa", "CR_In_Phags_pa"), + INSAURASHTRA("insaurashtra", "CR_In_Saurashtra"), + INDEVANAGARIEXTENDED("indevanagariextended", "CR_In_Devanagari_Extended"), + INKAYAHLI("inkayahli", "CR_In_Kayah_Li"), + INREJANG("inrejang", "CR_In_Rejang"), + INHANGULJAMOEXTENDEDA("inhanguljamoextendeda", "CR_In_Hangul_Jamo_Extended_A"), + INJAVANESE("injavanese", "CR_In_Javanese"), + INMYANMAREXTENDEDB("inmyanmarextendedb", "CR_In_Myanmar_Extended_B"), + INCHAM("incham", "CR_In_Cham"), + INMYANMAREXTENDEDA("inmyanmarextendeda", "CR_In_Myanmar_Extended_A"), + INTAIVIET("intaiviet", "CR_In_Tai_Viet"), + INMEETEIMAYEKEXTENSIONS("inmeeteimayekextensions", "CR_In_Meetei_Mayek_Extensions"), + INETHIOPICEXTENDEDA("inethiopicextendeda", "CR_In_Ethiopic_Extended_A"), + INLATINEXTENDEDE("inlatinextendede", "CR_In_Latin_Extended_E"), + INCHEROKEESUPPLEMENT("incherokeesupplement", "CR_In_Cherokee_Supplement"), + INMEETEIMAYEK("inmeeteimayek", "CR_In_Meetei_Mayek"), + INHANGULSYLLABLES("inhangulsyllables", "CR_In_Hangul_Syllables"), + INHANGULJAMOEXTENDEDB("inhanguljamoextendedb", "CR_In_Hangul_Jamo_Extended_B"), + INHIGHSURROGATES("inhighsurrogates", "CR_In_High_Surrogates"), + INHIGHPRIVATEUSESURROGATES("inhighprivateusesurrogates", "CR_In_High_Private_Use_Surrogates"), + INLOWSURROGATES("inlowsurrogates", "CR_In_Low_Surrogates"), + INPRIVATEUSEAREA("inprivateusearea", "CR_In_Private_Use_Area"), + INCJKCOMPATIBILITYIDEOGRAPHS("incjkcompatibilityideographs", "CR_In_CJK_Compatibility_Ideographs"), + INALPHABETICPRESENTATIONFORMS("inalphabeticpresentationforms", "CR_In_Alphabetic_Presentation_Forms"), + INARABICPRESENTATIONFORMSA("inarabicpresentationformsa", "CR_In_Arabic_Presentation_Forms_A"), + INVARIATIONSELECTORS("invariationselectors", "CR_In_Variation_Selectors"), + INVERTICALFORMS("inverticalforms", "CR_In_Vertical_Forms"), + INCOMBININGHALFMARKS("incombininghalfmarks", "CR_In_Combining_Half_Marks"), + INCJKCOMPATIBILITYFORMS("incjkcompatibilityforms", "CR_In_CJK_Compatibility_Forms"), + INSMALLFORMVARIANTS("insmallformvariants", "CR_In_Small_Form_Variants"), + INARABICPRESENTATIONFORMSB("inarabicpresentationformsb", "CR_In_Arabic_Presentation_Forms_B"), + INHALFWIDTHANDFULLWIDTHFORMS("inhalfwidthandfullwidthforms", "CR_In_Halfwidth_and_Fullwidth_Forms"), + INSPECIALS("inspecials", "CR_In_Specials"), + INLINEARBSYLLABARY("inlinearbsyllabary", "CR_In_Linear_B_Syllabary"), + INLINEARBIDEOGRAMS("inlinearbideograms", "CR_In_Linear_B_Ideograms"), + INAEGEANNUMBERS("inaegeannumbers", "CR_In_Aegean_Numbers"), + INANCIENTGREEKNUMBERS("inancientgreeknumbers", "CR_In_Ancient_Greek_Numbers"), + INANCIENTSYMBOLS("inancientsymbols", "CR_In_Ancient_Symbols"), + INPHAISTOSDISC("inphaistosdisc", "CR_In_Phaistos_Disc"), + INLYCIAN("inlycian", "CR_In_Lycian"), + INCARIAN("incarian", "CR_In_Carian"), + INCOPTICEPACTNUMBERS("incopticepactnumbers", "CR_In_Coptic_Epact_Numbers"), + INOLDITALIC("inolditalic", "CR_In_Old_Italic"), + INGOTHIC("ingothic", "CR_In_Gothic"), + INOLDPERMIC("inoldpermic", "CR_In_Old_Permic"), + INUGARITIC("inugaritic", "CR_In_Ugaritic"), + INOLDPERSIAN("inoldpersian", "CR_In_Old_Persian"), + DESERET("deseret", "CR_Deseret"), + DSRT("dsrt", "CR_Deseret"), + SHAVIAN("shavian", "CR_Shavian"), + SHAW("shaw", "CR_Shavian"), + INOSMANYA("inosmanya", "CR_In_Osmanya"), + INOSAGE("inosage", "CR_In_Osage"), + INELBASAN("inelbasan", "CR_In_Elbasan"), + INCAUCASIANALBANIAN("incaucasianalbanian", "CR_In_Caucasian_Albanian"), + INVITHKUQI("invithkuqi", "CR_In_Vithkuqi"), + INTODHRI("intodhri", "CR_In_Todhri"), + INLINEARA("inlineara", "CR_In_Linear_A"), + INLATINEXTENDEDF("inlatinextendedf", "CR_In_Latin_Extended_F"), + INCYPRIOTSYLLABARY("incypriotsyllabary", "CR_In_Cypriot_Syllabary"), + INIMPERIALARAMAIC("inimperialaramaic", "CR_In_Imperial_Aramaic"), + PALMYRENE("palmyrene", "CR_Palmyrene"), + PALM("palm", "CR_Palmyrene"), + INNABATAEAN("innabataean", "CR_In_Nabataean"), + INHATRAN("inhatran", "CR_In_Hatran"), + INPHOENICIAN("inphoenician", "CR_In_Phoenician"), + INLYDIAN("inlydian", "CR_In_Lydian"), + INSIDETIC("insidetic", "CR_In_Sidetic"), + MEROITICHIEROGLYPHS("meroitichieroglyphs", "CR_Meroitic_Hieroglyphs"), + MERO("mero", "CR_Meroitic_Hieroglyphs"), + INMEROITICCURSIVE("inmeroiticcursive", "CR_In_Meroitic_Cursive"), + INKHAROSHTHI("inkharoshthi", "CR_In_Kharoshthi"), + OLDSOUTHARABIAN("oldsoutharabian", "CR_Old_South_Arabian"), + SARB("sarb", "CR_Old_South_Arabian"), + OLDNORTHARABIAN("oldnortharabian", "CR_Old_North_Arabian"), + NARB("narb", "CR_Old_North_Arabian"), + INMANICHAEAN("inmanichaean", "CR_In_Manichaean"), + INAVESTAN("inavestan", "CR_In_Avestan"), + ININSCRIPTIONALPARTHIAN("ininscriptionalparthian", "CR_In_Inscriptional_Parthian"), + ININSCRIPTIONALPAHLAVI("ininscriptionalpahlavi", "CR_In_Inscriptional_Pahlavi"), + INPSALTERPAHLAVI("inpsalterpahlavi", "CR_In_Psalter_Pahlavi"), + INOLDTURKIC("inoldturkic", "CR_In_Old_Turkic"), + INOLDHUNGARIAN("inoldhungarian", "CR_In_Old_Hungarian"), + INHANIFIROHINGYA("inhanifirohingya", "CR_In_Hanifi_Rohingya"), + INGARAY("ingaray", "CR_In_Garay"), + INRUMINUMERALSYMBOLS("inruminumeralsymbols", "CR_In_Rumi_Numeral_Symbols"), + INYEZIDI("inyezidi", "CR_In_Yezidi"), + INARABICEXTENDEDC("inarabicextendedc", "CR_In_Arabic_Extended_C"), + INOLDSOGDIAN("inoldsogdian", "CR_In_Old_Sogdian"), + INSOGDIAN("insogdian", "CR_In_Sogdian"), + INOLDUYGHUR("inolduyghur", "CR_In_Old_Uyghur"), + INCHORASMIAN("inchorasmian", "CR_In_Chorasmian"), + INELYMAIC("inelymaic", "CR_In_Elymaic"), + INBRAHMI("inbrahmi", "CR_In_Brahmi"), + INKAITHI("inkaithi", "CR_In_Kaithi"), + INSORASOMPENG("insorasompeng", "CR_In_Sora_Sompeng"), + INCHAKMA("inchakma", "CR_In_Chakma"), + INMAHAJANI("inmahajani", "CR_In_Mahajani"), + INSHARADA("insharada", "CR_In_Sharada"), + INSINHALAARCHAICNUMBERS("insinhalaarchaicnumbers", "CR_In_Sinhala_Archaic_Numbers"), + INKHOJKI("inkhojki", "CR_In_Khojki"), + INMULTANI("inmultani", "CR_In_Multani"), + INKHUDAWADI("inkhudawadi", "CR_In_Khudawadi"), + INGRANTHA("ingrantha", "CR_In_Grantha"), + INTULUTIGALARI("intulutigalari", "CR_In_Tulu_Tigalari"), + INNEWA("innewa", "CR_In_Newa"), + INTIRHUTA("intirhuta", "CR_In_Tirhuta"), + INSIDDHAM("insiddham", "CR_In_Siddham"), + INMODI("inmodi", "CR_In_Modi"), + INMONGOLIANSUPPLEMENT("inmongoliansupplement", "CR_In_Mongolian_Supplement"), + INTAKRI("intakri", "CR_In_Takri"), + INMYANMAREXTENDEDC("inmyanmarextendedc", "CR_In_Myanmar_Extended_C"), + INAHOM("inahom", "CR_In_Ahom"), + INDOGRA("indogra", "CR_In_Dogra"), + INWARANGCITI("inwarangciti", "CR_In_Warang_Citi"), + INDIVESAKURU("indivesakuru", "CR_In_Dives_Akuru"), + INNANDINAGARI("innandinagari", "CR_In_Nandinagari"), + INZANABAZARSQUARE("inzanabazarsquare", "CR_In_Zanabazar_Square"), + INSOYOMBO("insoyombo", "CR_In_Soyombo"), + INUNIFIEDCANADIANABORIGINALSYLLABICSEXTENDEDA("inunifiedcanadianaboriginalsyllabicsextendeda", "CR_In_Unified_Canadian_Aboriginal_Syllabics_Extended_A"), + INPAUCINHAU("inpaucinhau", "CR_In_Pau_Cin_Hau"), + INDEVANAGARIEXTENDEDA("indevanagariextendeda", "CR_In_Devanagari_Extended_A"), + INSHARADASUPPLEMENT("insharadasupplement", "CR_In_Sharada_Supplement"), + INSUNUWAR("insunuwar", "CR_In_Sunuwar"), + INBHAIKSUKI("inbhaiksuki", "CR_In_Bhaiksuki"), + INMARCHEN("inmarchen", "CR_In_Marchen"), + INMASARAMGONDI("inmasaramgondi", "CR_In_Masaram_Gondi"), + INGUNJALAGONDI("ingunjalagondi", "CR_In_Gunjala_Gondi"), + INTOLONGSIKI("intolongsiki", "CR_In_Tolong_Siki"), + INMAKASAR("inmakasar", "CR_In_Makasar"), + INKAWI("inkawi", "CR_In_Kawi"), + INLISUSUPPLEMENT("inlisusupplement", "CR_In_Lisu_Supplement"), + INTAMILSUPPLEMENT("intamilsupplement", "CR_In_Tamil_Supplement"), + INCUNEIFORM("incuneiform", "CR_In_Cuneiform"), + INCUNEIFORMNUMBERSANDPUNCTUATION("incuneiformnumbersandpunctuation", "CR_In_Cuneiform_Numbers_and_Punctuation"), + INEARLYDYNASTICCUNEIFORM("inearlydynasticcuneiform", "CR_In_Early_Dynastic_Cuneiform"), + INCYPROMINOAN("incyprominoan", "CR_In_Cypro_Minoan"), + INEGYPTIANHIEROGLYPHS("inegyptianhieroglyphs", "CR_In_Egyptian_Hieroglyphs"), + INEGYPTIANHIEROGLYPHFORMATCONTROLS("inegyptianhieroglyphformatcontrols", "CR_In_Egyptian_Hieroglyph_Format_Controls"), + INEGYPTIANHIEROGLYPHSEXTENDEDA("inegyptianhieroglyphsextendeda", "CR_In_Egyptian_Hieroglyphs_Extended_A"), + INANATOLIANHIEROGLYPHS("inanatolianhieroglyphs", "CR_In_Anatolian_Hieroglyphs"), + INGURUNGKHEMA("ingurungkhema", "CR_In_Gurung_Khema"), + INBAMUMSUPPLEMENT("inbamumsupplement", "CR_In_Bamum_Supplement"), + INMRO("inmro", "CR_In_Mro"), + INTANGSA("intangsa", "CR_In_Tangsa"), + INBASSAVAH("inbassavah", "CR_In_Bassa_Vah"), + INPAHAWHHMONG("inpahawhhmong", "CR_In_Pahawh_Hmong"), + INKIRATRAI("inkiratrai", "CR_In_Kirat_Rai"), + INMEDEFAIDRIN("inmedefaidrin", "CR_In_Medefaidrin"), + INBERIAERFE("inberiaerfe", "CR_In_Beria_Erfe"), + INMIAO("inmiao", "CR_In_Miao"), + INIDEOGRAPHICSYMBOLSANDPUNCTUATION("inideographicsymbolsandpunctuation", "CR_In_Ideographic_Symbols_and_Punctuation"), + INTANGUT("intangut", "CR_In_Tangut"), + INTANGUTCOMPONENTS("intangutcomponents", "CR_In_Tangut_Components"), + INKHITANSMALLSCRIPT("inkhitansmallscript", "CR_In_Khitan_Small_Script"), + INTANGUTSUPPLEMENT("intangutsupplement", "CR_In_Tangut_Supplement"), + INTANGUTCOMPONENTSSUPPLEMENT("intangutcomponentssupplement", "CR_In_Tangut_Components_Supplement"), + INKANAEXTENDEDB("inkanaextendedb", "CR_In_Kana_Extended_B"), + INKANASUPPLEMENT("inkanasupplement", "CR_In_Kana_Supplement"), + INKANAEXTENDEDA("inkanaextendeda", "CR_In_Kana_Extended_A"), + INSMALLKANAEXTENSION("insmallkanaextension", "CR_In_Small_Kana_Extension"), + INNUSHU("innushu", "CR_In_Nushu"), + INDUPLOYAN("induployan", "CR_In_Duployan"), + INSHORTHANDFORMATCONTROLS("inshorthandformatcontrols", "CR_In_Shorthand_Format_Controls"), + INSYMBOLSFORLEGACYCOMPUTINGSUPPLEMENT("insymbolsforlegacycomputingsupplement", "CR_In_Symbols_for_Legacy_Computing_Supplement"), + INMISCELLANEOUSSYMBOLSSUPPLEMENT("inmiscellaneoussymbolssupplement", "CR_In_Miscellaneous_Symbols_Supplement"), + INZNAMENNYMUSICALNOTATION("inznamennymusicalnotation", "CR_In_Znamenny_Musical_Notation"), + INBYZANTINEMUSICALSYMBOLS("inbyzantinemusicalsymbols", "CR_In_Byzantine_Musical_Symbols"), + INMUSICALSYMBOLS("inmusicalsymbols", "CR_In_Musical_Symbols"), + INANCIENTGREEKMUSICALNOTATION("inancientgreekmusicalnotation", "CR_In_Ancient_Greek_Musical_Notation"), + INKAKTOVIKNUMERALS("inkaktoviknumerals", "CR_In_Kaktovik_Numerals"), + INMAYANNUMERALS("inmayannumerals", "CR_In_Mayan_Numerals"), + INTAIXUANJINGSYMBOLS("intaixuanjingsymbols", "CR_In_Tai_Xuan_Jing_Symbols"), + INCOUNTINGRODNUMERALS("incountingrodnumerals", "CR_In_Counting_Rod_Numerals"), + INMATHEMATICALALPHANUMERICSYMBOLS("inmathematicalalphanumericsymbols", "CR_In_Mathematical_Alphanumeric_Symbols"), + INSUTTONSIGNWRITING("insuttonsignwriting", "CR_In_Sutton_SignWriting"), + INLATINEXTENDEDG("inlatinextendedg", "CR_In_Latin_Extended_G"), + INGLAGOLITICSUPPLEMENT("inglagoliticsupplement", "CR_In_Glagolitic_Supplement"), + INCYRILLICEXTENDEDD("incyrillicextendedd", "CR_In_Cyrillic_Extended_D"), + INNYIAKENGPUACHUEHMONG("innyiakengpuachuehmong", "CR_In_Nyiakeng_Puachue_Hmong"), + INTOTO("intoto", "CR_In_Toto"), + INWANCHO("inwancho", "CR_In_Wancho"), + INNAGMUNDARI("innagmundari", "CR_In_Nag_Mundari"), + INOLONAL("inolonal", "CR_In_Ol_Onal"), + INTAIYO("intaiyo", "CR_In_Tai_Yo"), + INETHIOPICEXTENDEDB("inethiopicextendedb", "CR_In_Ethiopic_Extended_B"), + INMENDEKIKAKUI("inmendekikakui", "CR_In_Mende_Kikakui"), + INADLAM("inadlam", "CR_In_Adlam"), + ININDICSIYAQNUMBERS("inindicsiyaqnumbers", "CR_In_Indic_Siyaq_Numbers"), + INOTTOMANSIYAQNUMBERS("inottomansiyaqnumbers", "CR_In_Ottoman_Siyaq_Numbers"), + INARABICMATHEMATICALALPHABETICSYMBOLS("inarabicmathematicalalphabeticsymbols", "CR_In_Arabic_Mathematical_Alphabetic_Symbols"), + INMAHJONGTILES("inmahjongtiles", "CR_In_Mahjong_Tiles"), + INDOMINOTILES("indominotiles", "CR_In_Domino_Tiles"), + INPLAYINGCARDS("inplayingcards", "CR_In_Playing_Cards"), + INENCLOSEDALPHANUMERICSUPPLEMENT("inenclosedalphanumericsupplement", "CR_In_Enclosed_Alphanumeric_Supplement"), + INENCLOSEDIDEOGRAPHICSUPPLEMENT("inenclosedideographicsupplement", "CR_In_Enclosed_Ideographic_Supplement"), + INMISCELLANEOUSSYMBOLSANDPICTOGRAPHS("inmiscellaneoussymbolsandpictographs", "CR_In_Miscellaneous_Symbols_and_Pictographs"), + INEMOTICONS("inemoticons", "CR_In_Emoticons"), + INORNAMENTALDINGBATS("inornamentaldingbats", "CR_In_Ornamental_Dingbats"), + INTRANSPORTANDMAPSYMBOLS("intransportandmapsymbols", "CR_In_Transport_and_Map_Symbols"), + INALCHEMICALSYMBOLS("inalchemicalsymbols", "CR_In_Alchemical_Symbols"), + INGEOMETRICSHAPESEXTENDED("ingeometricshapesextended", "CR_In_Geometric_Shapes_Extended"), + INSUPPLEMENTALARROWSC("insupplementalarrowsc", "CR_In_Supplemental_Arrows_C"), + INSUPPLEMENTALSYMBOLSANDPICTOGRAPHS("insupplementalsymbolsandpictographs", "CR_In_Supplemental_Symbols_and_Pictographs"), + INCHESSSYMBOLS("inchesssymbols", "CR_In_Chess_Symbols"), + INSYMBOLSANDPICTOGRAPHSEXTENDEDA("insymbolsandpictographsextendeda", "CR_In_Symbols_and_Pictographs_Extended_A"), + INSYMBOLSFORLEGACYCOMPUTING("insymbolsforlegacycomputing", "CR_In_Symbols_for_Legacy_Computing"), + INCJKUNIFIEDIDEOGRAPHSEXTENSIONB("incjkunifiedideographsextensionb", "CR_In_CJK_Unified_Ideographs_Extension_B"), + INCJKUNIFIEDIDEOGRAPHSEXTENSIONC("incjkunifiedideographsextensionc", "CR_In_CJK_Unified_Ideographs_Extension_C"), + INCJKUNIFIEDIDEOGRAPHSEXTENSIOND("incjkunifiedideographsextensiond", "CR_In_CJK_Unified_Ideographs_Extension_D"), + INCJKUNIFIEDIDEOGRAPHSEXTENSIONE("incjkunifiedideographsextensione", "CR_In_CJK_Unified_Ideographs_Extension_E"), + INCJKUNIFIEDIDEOGRAPHSEXTENSIONF("incjkunifiedideographsextensionf", "CR_In_CJK_Unified_Ideographs_Extension_F"), + INCJKUNIFIEDIDEOGRAPHSEXTENSIONI("incjkunifiedideographsextensioni", "CR_In_CJK_Unified_Ideographs_Extension_I"), + INCJKCOMPATIBILITYIDEOGRAPHSSUPPLEMENT("incjkcompatibilityideographssupplement", "CR_In_CJK_Compatibility_Ideographs_Supplement"), + INCJKUNIFIEDIDEOGRAPHSEXTENSIONG("incjkunifiedideographsextensiong", "CR_In_CJK_Unified_Ideographs_Extension_G"), + INCJKUNIFIEDIDEOGRAPHSEXTENSIONH("incjkunifiedideographsextensionh", "CR_In_CJK_Unified_Ideographs_Extension_H"), + INCJKUNIFIEDIDEOGRAPHSEXTENSIONJ("incjkunifiedideographsextensionj", "CR_In_CJK_Unified_Ideographs_Extension_J"), + INTAGS("intags", "CR_In_Tags"), + INVARIATIONSELECTORSSUPPLEMENT("invariationselectorssupplement", "CR_In_Variation_Selectors_Supplement"), + INSUPPLEMENTARYPRIVATEUSEAREAA("insupplementaryprivateuseareaa", "CR_In_Supplementary_Private_Use_Area_A"), + INSUPPLEMENTARYPRIVATEUSEAREAB("insupplementaryprivateuseareab", "CR_In_Supplementary_Private_Use_Area_B"), + INNOBLOCK("innoblock", "CR_In_No_Block"); + + private final String table; + final byte[]name; + private int[]range; + + private UnicodeCodeRange(String name, String table) { + this.table = table; + this.name = name.getBytes(); + } + + int[]getRange() { + if (range == null) range = ArrayReader.readIntArray(table); + return range; + } + + public boolean contains(int code) { + return CodeRange.isInCodeRange(range, code); + } + + public int getCType() { + return ordinal(); + } + + static final UnicodeCodeRange[]CodeRangeTable = UnicodeCodeRange.values(); + static final int MAX_WORD_LENGTH = 45; +} diff --git a/src/org/jcodings/unicode/UnicodeEncoding.java b/src/org/jcodings/unicode/UnicodeEncoding.java index f856acd5..cd75ad80 100644 --- a/src/org/jcodings/unicode/UnicodeEncoding.java +++ b/src/org/jcodings/unicode/UnicodeEncoding.java @@ -38,7 +38,7 @@ import org.jcodings.util.IntHash; public abstract class UnicodeEncoding extends MultiByteEncoding { - private static final int PROPERTY_NAME_MAX_SIZE = UnicodeProperties.MAX_WORD_LENGTH + 1; + private static final int PROPERTY_NAME_MAX_SIZE = UnicodeCodeRange.MAX_WORD_LENGTH + 1; static final int I_WITH_DOT_ABOVE = 0x0130; static final int DOTLESS_i = 0x0131; static final int DOT_ABOVE = 0x0307; @@ -68,17 +68,21 @@ public boolean isCodeCType(int code, int ctype) { if (code < 256) return isCodeCTypeInternal(code, ctype); } - if (ctype > UnicodeProperties.CodeRangeTable.length) throw new InternalError(ErrorMessages.ERR_TYPE_BUG); + if (ctype > UnicodeCodeRange.CodeRangeTable.length) throw new InternalError(ErrorMessages.ERR_TYPE_BUG); - return CodeRange.isInCodeRange(UnicodeProperties.CodeRangeTable[ctype].getRange(), code); + return CodeRange.isInCodeRange(UnicodeCodeRange.CodeRangeTable[ctype].getRange(), code); } + public static boolean isInCodeRange(UnicodeCodeRange range, int code) { + return CodeRange.isInCodeRange(range.getRange(), code); + } + // onigenc_unicode_ctype_code_range protected final int[]ctypeCodeRange(int ctype) { - if (ctype >= UnicodeProperties.CodeRangeTable.length) throw new InternalError(ErrorMessages.ERR_TYPE_BUG); + if (ctype >= UnicodeCodeRange.CodeRangeTable.length) throw new InternalError(ErrorMessages.ERR_TYPE_BUG); - return UnicodeProperties.CodeRangeTable[ctype].getRange(); + return UnicodeCodeRange.CodeRangeTable[ctype].getRange(); } // onigenc_unicode_property_name_to_ctype @@ -407,7 +411,7 @@ public void applyAllCaseFold(int flag, ApplyAllCaseFoldFunction fun, Object arg) } } - if (items == null || n == 0) return EMPTY_FOLD_CODES; + if (items == null || n == 0) return CaseFoldCodeItem.EMPTY_FOLD_CODES; if (n < items.length) { CaseFoldCodeItem [] tmp = new CaseFoldCodeItem[n]; System.arraycopy(items, 0, tmp, 0, n); @@ -461,9 +465,12 @@ public final int caseMap(IntHolder flagP, byte[] bytes, IntHolder pp, int end, b flags |= Config.CASE_MODIFIED; code = 'I'; } - } else if ((folded = CaseFold.Values.get(code)) != null) { - if ((flags & Config.CASE_TITLECASE) != 0 && (folded.flags & Config.CASE_IS_TITLECASE) != 0) { - + } else if ((folded = CaseFold.Values.get(code)) != null) { /* data about character found in CaseFold_Table */ + if ((flags & Config.CASE_TITLECASE) != 0 && code >= 0x1C90 && code <= 0x1CBF) { /* Georgian MTAVRULI */ + flags |= Config.CASE_MODIFIED; + code += 0x10D0 - 0x1C90; + } else if ((flags & Config.CASE_TITLECASE) != 0 && (folded.flags & Config.CASE_IS_TITLECASE) != 0) { /* Titlecase needed, but already Titlecase */ + /* already Titlecase, no changes needed */ } else if ((flags & folded.flags) != 0) { final int[]codes; final int start; @@ -504,9 +511,13 @@ public final int caseMap(IntHolder flagP, byte[] bytes, IntHolder pp, int end, b code = codes[i]; } } - } else if ((folded = CaseUnfold11.Values.get(code)) != null && (flags & folded.flags) != 0) { - flags |= Config.CASE_MODIFIED; - code = folded.codes[(flags & folded.flags & Config.CASE_TITLECASE) != 0 ? 1 : 0]; + } else if ((folded = CaseUnfold11.Values.get(code)) != null) { /* data about character found in CaseUnfold_11_Table */ + if ((flags & Config.CASE_TITLECASE) != 0 && (folded.flags & Config.CASE_IS_TITLECASE) != 0) { /* Titlecase needed, but already Titlecase */ + /* already Titlecase, no changes needed */ + } else if ((flags & folded.flags) != 0) { /* needs and data availability match */ + flags |= Config.CASE_MODIFIED; + code = folded.codes[(flags & folded.flags & Config.CASE_TITLECASE) != 0 ? 1 : 0]; + } } } toP += codeToMbc(code, to, toP); @@ -553,29 +564,13 @@ public final int caseMap(IntHolder flagP, byte[] bytes, IntHolder pp, int end, b 0x30e2, 0x30e2, 0x30e2, 0x30e2, 0x30e2, 0x30e2, 0x30e2, 0x30e2 }; - static final class CodeRangeEntry { - final String table; - final byte[]name; - int[]range; - - CodeRangeEntry(String name, String table) { - this.table = table; - this.name = name.getBytes(); - } - - public int[]getRange() { - if (range == null) range = ArrayReader.readIntArray(table); - return range; - } - } - static class CTypeName { private static final CaseInsensitiveBytesHash Values = initializeCTypeNameTable(); private static CaseInsensitiveBytesHash initializeCTypeNameTable() { CaseInsensitiveBytesHash table = new CaseInsensitiveBytesHash(); - for (int i = 0; i < UnicodeProperties.CodeRangeTable.length; i++) { - table.putDirect(UnicodeProperties.CodeRangeTable[i].name, i); + for (int i = 0; i < UnicodeCodeRange.CodeRangeTable.length; i++) { + table.putDirect(UnicodeCodeRange.CodeRangeTable[i].name, i); } return table; } diff --git a/src/org/jcodings/unicode/UnicodeProperties.java b/src/org/jcodings/unicode/UnicodeProperties.java deleted file mode 100644 index 55b3d1cc..00000000 --- a/src/org/jcodings/unicode/UnicodeProperties.java +++ /dev/null @@ -1,824 +0,0 @@ -/* - * Permission is hereby granted, free of charge, to any person obtaining a copy of - * this software and associated documentation files (the "Software"), to deal in - * the Software without restriction, including without limitation the rights to - * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies - * of the Software, and to permit persons to whom the Software is furnished to do - * so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ -package org.jcodings.unicode; - -import org.jcodings.unicode.UnicodeEncoding.CodeRangeEntry; - -public class UnicodeProperties { - static final CodeRangeEntry[]CodeRangeTable = new CodeRangeEntry[] { - new CodeRangeEntry("newline", "CR_NEWLINE"), - new CodeRangeEntry("alpha", "CR_Alpha"), - new CodeRangeEntry("blank", "CR_Blank"), - new CodeRangeEntry("cntrl", "CR_Cntrl"), - new CodeRangeEntry("digit", "CR_Digit"), - new CodeRangeEntry("graph", "CR_Graph"), - new CodeRangeEntry("lower", "CR_Lower"), - new CodeRangeEntry("print", "CR_Print"), - new CodeRangeEntry("xposixpunct", "CR_XPosixPunct"), - new CodeRangeEntry("space", "CR_Space"), - new CodeRangeEntry("upper", "CR_Upper"), - new CodeRangeEntry("xdigit", "CR_XDigit"), - new CodeRangeEntry("word", "CR_Word"), - new CodeRangeEntry("alnum", "CR_Alnum"), - new CodeRangeEntry("ascii", "CR_ASCII"), - new CodeRangeEntry("punct", "CR_Punct"), - new CodeRangeEntry("any", "CR_Any"), - new CodeRangeEntry("assigned", "CR_Assigned"), - new CodeRangeEntry("c", "CR_C"), - new CodeRangeEntry("other", "CR_C"), - new CodeRangeEntry("cc", "CR_Cntrl"), - new CodeRangeEntry("control", "CR_Cntrl"), - new CodeRangeEntry("cf", "CR_Cf"), - new CodeRangeEntry("format", "CR_Cf"), - new CodeRangeEntry("cn", "CR_Cn"), - new CodeRangeEntry("unassigned", "CR_Cn"), - new CodeRangeEntry("co", "CR_Co"), - new CodeRangeEntry("privateuse", "CR_Co"), - new CodeRangeEntry("cs", "CR_Cs"), - new CodeRangeEntry("surrogate", "CR_Cs"), - new CodeRangeEntry("l", "CR_L"), - new CodeRangeEntry("letter", "CR_L"), - new CodeRangeEntry("lc", "CR_LC"), - new CodeRangeEntry("casedletter", "CR_LC"), - new CodeRangeEntry("ll", "CR_Ll"), - new CodeRangeEntry("lowercaseletter", "CR_Ll"), - new CodeRangeEntry("lm", "CR_Lm"), - new CodeRangeEntry("modifierletter", "CR_Lm"), - new CodeRangeEntry("lo", "CR_Lo"), - new CodeRangeEntry("otherletter", "CR_Lo"), - new CodeRangeEntry("lt", "CR_Lt"), - new CodeRangeEntry("titlecaseletter", "CR_Lt"), - new CodeRangeEntry("lu", "CR_Lu"), - new CodeRangeEntry("uppercaseletter", "CR_Lu"), - new CodeRangeEntry("m", "CR_M"), - new CodeRangeEntry("mark", "CR_M"), - new CodeRangeEntry("combiningmark", "CR_M"), - new CodeRangeEntry("mc", "CR_Mc"), - new CodeRangeEntry("spacingmark", "CR_Mc"), - new CodeRangeEntry("me", "CR_Me"), - new CodeRangeEntry("enclosingmark", "CR_Me"), - new CodeRangeEntry("mn", "CR_Mn"), - new CodeRangeEntry("nonspacingmark", "CR_Mn"), - new CodeRangeEntry("n", "CR_N"), - new CodeRangeEntry("number", "CR_N"), - new CodeRangeEntry("nd", "CR_Digit"), - new CodeRangeEntry("decimalnumber", "CR_Digit"), - new CodeRangeEntry("nl", "CR_Nl"), - new CodeRangeEntry("letternumber", "CR_Nl"), - new CodeRangeEntry("no", "CR_No"), - new CodeRangeEntry("othernumber", "CR_No"), - new CodeRangeEntry("p", "CR_Punct"), - new CodeRangeEntry("punctuation", "CR_Punct"), - new CodeRangeEntry("pc", "CR_Pc"), - new CodeRangeEntry("connectorpunctuation", "CR_Pc"), - new CodeRangeEntry("pd", "CR_Pd"), - new CodeRangeEntry("dashpunctuation", "CR_Pd"), - new CodeRangeEntry("pe", "CR_Pe"), - new CodeRangeEntry("closepunctuation", "CR_Pe"), - new CodeRangeEntry("pf", "CR_Pf"), - new CodeRangeEntry("finalpunctuation", "CR_Pf"), - new CodeRangeEntry("pi", "CR_Pi"), - new CodeRangeEntry("initialpunctuation", "CR_Pi"), - new CodeRangeEntry("po", "CR_Po"), - new CodeRangeEntry("otherpunctuation", "CR_Po"), - new CodeRangeEntry("ps", "CR_Ps"), - new CodeRangeEntry("openpunctuation", "CR_Ps"), - new CodeRangeEntry("s", "CR_S"), - new CodeRangeEntry("symbol", "CR_S"), - new CodeRangeEntry("sc", "CR_Sc"), - new CodeRangeEntry("currencysymbol", "CR_Sc"), - new CodeRangeEntry("sk", "CR_Sk"), - new CodeRangeEntry("modifiersymbol", "CR_Sk"), - new CodeRangeEntry("sm", "CR_Sm"), - new CodeRangeEntry("mathsymbol", "CR_Sm"), - new CodeRangeEntry("so", "CR_So"), - new CodeRangeEntry("othersymbol", "CR_So"), - new CodeRangeEntry("z", "CR_Z"), - new CodeRangeEntry("separator", "CR_Z"), - new CodeRangeEntry("zl", "CR_Zl"), - new CodeRangeEntry("lineseparator", "CR_Zl"), - new CodeRangeEntry("zp", "CR_Zp"), - new CodeRangeEntry("paragraphseparator", "CR_Zp"), - new CodeRangeEntry("zs", "CR_Zs"), - new CodeRangeEntry("spaceseparator", "CR_Zs"), - new CodeRangeEntry("math", "CR_Math"), - new CodeRangeEntry("alphabetic", "CR_Alpha"), - new CodeRangeEntry("lowercase", "CR_Lower"), - new CodeRangeEntry("uppercase", "CR_Upper"), - new CodeRangeEntry("cased", "CR_Cased"), - new CodeRangeEntry("caseignorable", "CR_Case_Ignorable"), - new CodeRangeEntry("ci", "CR_Case_Ignorable"), - new CodeRangeEntry("changeswhenlowercased", "CR_Changes_When_Lowercased"), - new CodeRangeEntry("cwl", "CR_Changes_When_Lowercased"), - new CodeRangeEntry("changeswhenuppercased", "CR_Changes_When_Uppercased"), - new CodeRangeEntry("cwu", "CR_Changes_When_Uppercased"), - new CodeRangeEntry("changeswhentitlecased", "CR_Changes_When_Titlecased"), - new CodeRangeEntry("cwt", "CR_Changes_When_Titlecased"), - new CodeRangeEntry("changeswhencasefolded", "CR_Changes_When_Casefolded"), - new CodeRangeEntry("cwcf", "CR_Changes_When_Casefolded"), - new CodeRangeEntry("changeswhencasemapped", "CR_Changes_When_Casemapped"), - new CodeRangeEntry("cwcm", "CR_Changes_When_Casemapped"), - new CodeRangeEntry("idstart", "CR_ID_Start"), - new CodeRangeEntry("ids", "CR_ID_Start"), - new CodeRangeEntry("idcontinue", "CR_ID_Continue"), - new CodeRangeEntry("idc", "CR_ID_Continue"), - new CodeRangeEntry("xidstart", "CR_XID_Start"), - new CodeRangeEntry("xids", "CR_XID_Start"), - new CodeRangeEntry("xidcontinue", "CR_XID_Continue"), - new CodeRangeEntry("xidc", "CR_XID_Continue"), - new CodeRangeEntry("defaultignorablecodepoint", "CR_Default_Ignorable_Code_Point"), - new CodeRangeEntry("di", "CR_Default_Ignorable_Code_Point"), - new CodeRangeEntry("graphemeclusterbreak=extend", "CR_Grapheme_Extend"), - new CodeRangeEntry("graphemebase", "CR_Grapheme_Base"), - new CodeRangeEntry("grbase", "CR_Grapheme_Base"), - new CodeRangeEntry("graphemelink", "CR_Grapheme_Link"), - new CodeRangeEntry("grlink", "CR_Grapheme_Link"), - new CodeRangeEntry("common", "CR_Common"), - new CodeRangeEntry("zyyy", "CR_Common"), - new CodeRangeEntry("latin", "CR_Latin"), - new CodeRangeEntry("latn", "CR_Latin"), - new CodeRangeEntry("greek", "CR_Greek"), - new CodeRangeEntry("grek", "CR_Greek"), - new CodeRangeEntry("cyrillic", "CR_Cyrillic"), - new CodeRangeEntry("cyrl", "CR_Cyrillic"), - new CodeRangeEntry("armenian", "CR_Armenian"), - new CodeRangeEntry("armn", "CR_Armenian"), - new CodeRangeEntry("hebrew", "CR_Hebrew"), - new CodeRangeEntry("hebr", "CR_Hebrew"), - new CodeRangeEntry("arabic", "CR_Arabic"), - new CodeRangeEntry("arab", "CR_Arabic"), - new CodeRangeEntry("syriac", "CR_Syriac"), - new CodeRangeEntry("syrc", "CR_Syriac"), - new CodeRangeEntry("thaana", "CR_Thaana"), - new CodeRangeEntry("thaa", "CR_Thaana"), - new CodeRangeEntry("devanagari", "CR_Devanagari"), - new CodeRangeEntry("deva", "CR_Devanagari"), - new CodeRangeEntry("bengali", "CR_Bengali"), - new CodeRangeEntry("beng", "CR_Bengali"), - new CodeRangeEntry("gurmukhi", "CR_Gurmukhi"), - new CodeRangeEntry("guru", "CR_Gurmukhi"), - new CodeRangeEntry("gujarati", "CR_Gujarati"), - new CodeRangeEntry("gujr", "CR_Gujarati"), - new CodeRangeEntry("oriya", "CR_Oriya"), - new CodeRangeEntry("orya", "CR_Oriya"), - new CodeRangeEntry("tamil", "CR_Tamil"), - new CodeRangeEntry("taml", "CR_Tamil"), - new CodeRangeEntry("telugu", "CR_Telugu"), - new CodeRangeEntry("telu", "CR_Telugu"), - new CodeRangeEntry("kannada", "CR_Kannada"), - new CodeRangeEntry("knda", "CR_Kannada"), - new CodeRangeEntry("malayalam", "CR_Malayalam"), - new CodeRangeEntry("mlym", "CR_Malayalam"), - new CodeRangeEntry("sinhala", "CR_Sinhala"), - new CodeRangeEntry("sinh", "CR_Sinhala"), - new CodeRangeEntry("thai", "CR_Thai"), - new CodeRangeEntry("lao", "CR_Lao"), - new CodeRangeEntry("laoo", "CR_Lao"), - new CodeRangeEntry("tibetan", "CR_Tibetan"), - new CodeRangeEntry("tibt", "CR_Tibetan"), - new CodeRangeEntry("myanmar", "CR_Myanmar"), - new CodeRangeEntry("mymr", "CR_Myanmar"), - new CodeRangeEntry("georgian", "CR_Georgian"), - new CodeRangeEntry("geor", "CR_Georgian"), - new CodeRangeEntry("hangul", "CR_Hangul"), - new CodeRangeEntry("hang", "CR_Hangul"), - new CodeRangeEntry("ethiopic", "CR_Ethiopic"), - new CodeRangeEntry("ethi", "CR_Ethiopic"), - new CodeRangeEntry("cherokee", "CR_Cherokee"), - new CodeRangeEntry("cher", "CR_Cherokee"), - new CodeRangeEntry("canadianaboriginal", "CR_Canadian_Aboriginal"), - new CodeRangeEntry("cans", "CR_Canadian_Aboriginal"), - new CodeRangeEntry("ogham", "CR_Ogham"), - new CodeRangeEntry("ogam", "CR_Ogham"), - new CodeRangeEntry("runic", "CR_Runic"), - new CodeRangeEntry("runr", "CR_Runic"), - new CodeRangeEntry("khmer", "CR_Khmer"), - new CodeRangeEntry("khmr", "CR_Khmer"), - new CodeRangeEntry("mongolian", "CR_Mongolian"), - new CodeRangeEntry("mong", "CR_Mongolian"), - new CodeRangeEntry("hiragana", "CR_Hiragana"), - new CodeRangeEntry("hira", "CR_Hiragana"), - new CodeRangeEntry("katakana", "CR_Katakana"), - new CodeRangeEntry("kana", "CR_Katakana"), - new CodeRangeEntry("bopomofo", "CR_Bopomofo"), - new CodeRangeEntry("bopo", "CR_Bopomofo"), - new CodeRangeEntry("han", "CR_Han"), - new CodeRangeEntry("hani", "CR_Han"), - new CodeRangeEntry("yi", "CR_Yi"), - new CodeRangeEntry("yiii", "CR_Yi"), - new CodeRangeEntry("olditalic", "CR_Old_Italic"), - new CodeRangeEntry("ital", "CR_Old_Italic"), - new CodeRangeEntry("gothic", "CR_Gothic"), - new CodeRangeEntry("goth", "CR_Gothic"), - new CodeRangeEntry("indeseret", "CR_Deseret"), - new CodeRangeEntry("inherited", "CR_Inherited"), - new CodeRangeEntry("qaai", "CR_Inherited"), - new CodeRangeEntry("zinh", "CR_Inherited"), - new CodeRangeEntry("tagalog", "CR_Tagalog"), - new CodeRangeEntry("tglg", "CR_Tagalog"), - new CodeRangeEntry("hanunoo", "CR_Hanunoo"), - new CodeRangeEntry("hano", "CR_Hanunoo"), - new CodeRangeEntry("buhid", "CR_Buhid"), - new CodeRangeEntry("buhd", "CR_Buhid"), - new CodeRangeEntry("tagbanwa", "CR_Tagbanwa"), - new CodeRangeEntry("tagb", "CR_Tagbanwa"), - new CodeRangeEntry("limbu", "CR_Limbu"), - new CodeRangeEntry("limb", "CR_Limbu"), - new CodeRangeEntry("taile", "CR_Tai_Le"), - new CodeRangeEntry("tale", "CR_Tai_Le"), - new CodeRangeEntry("linearb", "CR_Linear_B"), - new CodeRangeEntry("linb", "CR_Linear_B"), - new CodeRangeEntry("ugaritic", "CR_Ugaritic"), - new CodeRangeEntry("ugar", "CR_Ugaritic"), - new CodeRangeEntry("inshavian", "CR_Shavian"), - new CodeRangeEntry("osmanya", "CR_Osmanya"), - new CodeRangeEntry("osma", "CR_Osmanya"), - new CodeRangeEntry("cypriot", "CR_Cypriot"), - new CodeRangeEntry("cprt", "CR_Cypriot"), - new CodeRangeEntry("inbraillepatterns", "CR_Braille"), - new CodeRangeEntry("buginese", "CR_Buginese"), - new CodeRangeEntry("bugi", "CR_Buginese"), - new CodeRangeEntry("coptic", "CR_Coptic"), - new CodeRangeEntry("qaac", "CR_Coptic"), - new CodeRangeEntry("copt", "CR_Coptic"), - new CodeRangeEntry("newtailue", "CR_New_Tai_Lue"), - new CodeRangeEntry("talu", "CR_New_Tai_Lue"), - new CodeRangeEntry("glagolitic", "CR_Glagolitic"), - new CodeRangeEntry("glag", "CR_Glagolitic"), - new CodeRangeEntry("tifinagh", "CR_Tifinagh"), - new CodeRangeEntry("tfng", "CR_Tifinagh"), - new CodeRangeEntry("sylotinagri", "CR_Syloti_Nagri"), - new CodeRangeEntry("sylo", "CR_Syloti_Nagri"), - new CodeRangeEntry("oldpersian", "CR_Old_Persian"), - new CodeRangeEntry("xpeo", "CR_Old_Persian"), - new CodeRangeEntry("kharoshthi", "CR_Kharoshthi"), - new CodeRangeEntry("khar", "CR_Kharoshthi"), - new CodeRangeEntry("balinese", "CR_Balinese"), - new CodeRangeEntry("bali", "CR_Balinese"), - new CodeRangeEntry("cuneiform", "CR_Cuneiform"), - new CodeRangeEntry("xsux", "CR_Cuneiform"), - new CodeRangeEntry("phoenician", "CR_Phoenician"), - new CodeRangeEntry("phnx", "CR_Phoenician"), - new CodeRangeEntry("phagspa", "CR_Phags_Pa"), - new CodeRangeEntry("phag", "CR_Phags_Pa"), - new CodeRangeEntry("nko", "CR_Nko"), - new CodeRangeEntry("nkoo", "CR_Nko"), - new CodeRangeEntry("sundanese", "CR_Sundanese"), - new CodeRangeEntry("sund", "CR_Sundanese"), - new CodeRangeEntry("lepcha", "CR_Lepcha"), - new CodeRangeEntry("lepc", "CR_Lepcha"), - new CodeRangeEntry("inolchiki", "CR_Ol_Chiki"), - new CodeRangeEntry("vai", "CR_Vai"), - new CodeRangeEntry("vaii", "CR_Vai"), - new CodeRangeEntry("saurashtra", "CR_Saurashtra"), - new CodeRangeEntry("saur", "CR_Saurashtra"), - new CodeRangeEntry("kayahli", "CR_Kayah_Li"), - new CodeRangeEntry("kali", "CR_Kayah_Li"), - new CodeRangeEntry("rejang", "CR_Rejang"), - new CodeRangeEntry("rjng", "CR_Rejang"), - new CodeRangeEntry("lycian", "CR_Lycian"), - new CodeRangeEntry("lyci", "CR_Lycian"), - new CodeRangeEntry("carian", "CR_Carian"), - new CodeRangeEntry("cari", "CR_Carian"), - new CodeRangeEntry("lydian", "CR_Lydian"), - new CodeRangeEntry("lydi", "CR_Lydian"), - new CodeRangeEntry("cham", "CR_Cham"), - new CodeRangeEntry("taitham", "CR_Tai_Tham"), - new CodeRangeEntry("lana", "CR_Tai_Tham"), - new CodeRangeEntry("taiviet", "CR_Tai_Viet"), - new CodeRangeEntry("tavt", "CR_Tai_Viet"), - new CodeRangeEntry("avestan", "CR_Avestan"), - new CodeRangeEntry("avst", "CR_Avestan"), - new CodeRangeEntry("egyptianhieroglyphs", "CR_Egyptian_Hieroglyphs"), - new CodeRangeEntry("egyp", "CR_Egyptian_Hieroglyphs"), - new CodeRangeEntry("samaritan", "CR_Samaritan"), - new CodeRangeEntry("samr", "CR_Samaritan"), - new CodeRangeEntry("inlisu", "CR_Lisu"), - new CodeRangeEntry("bamum", "CR_Bamum"), - new CodeRangeEntry("bamu", "CR_Bamum"), - new CodeRangeEntry("javanese", "CR_Javanese"), - new CodeRangeEntry("java", "CR_Javanese"), - new CodeRangeEntry("meeteimayek", "CR_Meetei_Mayek"), - new CodeRangeEntry("mtei", "CR_Meetei_Mayek"), - new CodeRangeEntry("imperialaramaic", "CR_Imperial_Aramaic"), - new CodeRangeEntry("armi", "CR_Imperial_Aramaic"), - new CodeRangeEntry("inoldsoutharabian", "CR_Old_South_Arabian"), - new CodeRangeEntry("inscriptionalparthian", "CR_Inscriptional_Parthian"), - new CodeRangeEntry("prti", "CR_Inscriptional_Parthian"), - new CodeRangeEntry("inscriptionalpahlavi", "CR_Inscriptional_Pahlavi"), - new CodeRangeEntry("phli", "CR_Inscriptional_Pahlavi"), - new CodeRangeEntry("oldturkic", "CR_Old_Turkic"), - new CodeRangeEntry("orkh", "CR_Old_Turkic"), - new CodeRangeEntry("kaithi", "CR_Kaithi"), - new CodeRangeEntry("kthi", "CR_Kaithi"), - new CodeRangeEntry("batak", "CR_Batak"), - new CodeRangeEntry("batk", "CR_Batak"), - new CodeRangeEntry("brahmi", "CR_Brahmi"), - new CodeRangeEntry("brah", "CR_Brahmi"), - new CodeRangeEntry("mandaic", "CR_Mandaic"), - new CodeRangeEntry("mand", "CR_Mandaic"), - new CodeRangeEntry("chakma", "CR_Chakma"), - new CodeRangeEntry("cakm", "CR_Chakma"), - new CodeRangeEntry("meroiticcursive", "CR_Meroitic_Cursive"), - new CodeRangeEntry("merc", "CR_Meroitic_Cursive"), - new CodeRangeEntry("inmeroitichieroglyphs", "CR_Meroitic_Hieroglyphs"), - new CodeRangeEntry("miao", "CR_Miao"), - new CodeRangeEntry("plrd", "CR_Miao"), - new CodeRangeEntry("sharada", "CR_Sharada"), - new CodeRangeEntry("shrd", "CR_Sharada"), - new CodeRangeEntry("sorasompeng", "CR_Sora_Sompeng"), - new CodeRangeEntry("sora", "CR_Sora_Sompeng"), - new CodeRangeEntry("takri", "CR_Takri"), - new CodeRangeEntry("takr", "CR_Takri"), - new CodeRangeEntry("caucasianalbanian", "CR_Caucasian_Albanian"), - new CodeRangeEntry("aghb", "CR_Caucasian_Albanian"), - new CodeRangeEntry("bassavah", "CR_Bassa_Vah"), - new CodeRangeEntry("bass", "CR_Bassa_Vah"), - new CodeRangeEntry("duployan", "CR_Duployan"), - new CodeRangeEntry("dupl", "CR_Duployan"), - new CodeRangeEntry("elbasan", "CR_Elbasan"), - new CodeRangeEntry("elba", "CR_Elbasan"), - new CodeRangeEntry("grantha", "CR_Grantha"), - new CodeRangeEntry("gran", "CR_Grantha"), - new CodeRangeEntry("pahawhhmong", "CR_Pahawh_Hmong"), - new CodeRangeEntry("hmng", "CR_Pahawh_Hmong"), - new CodeRangeEntry("khojki", "CR_Khojki"), - new CodeRangeEntry("khoj", "CR_Khojki"), - new CodeRangeEntry("lineara", "CR_Linear_A"), - new CodeRangeEntry("lina", "CR_Linear_A"), - new CodeRangeEntry("mahajani", "CR_Mahajani"), - new CodeRangeEntry("mahj", "CR_Mahajani"), - new CodeRangeEntry("manichaean", "CR_Manichaean"), - new CodeRangeEntry("mani", "CR_Manichaean"), - new CodeRangeEntry("mendekikakui", "CR_Mende_Kikakui"), - new CodeRangeEntry("mend", "CR_Mende_Kikakui"), - new CodeRangeEntry("modi", "CR_Modi"), - new CodeRangeEntry("mro", "CR_Mro"), - new CodeRangeEntry("mroo", "CR_Mro"), - new CodeRangeEntry("inoldnortharabian", "CR_Old_North_Arabian"), - new CodeRangeEntry("nabataean", "CR_Nabataean"), - new CodeRangeEntry("nbat", "CR_Nabataean"), - new CodeRangeEntry("inpalmyrene", "CR_Palmyrene"), - new CodeRangeEntry("paucinhau", "CR_Pau_Cin_Hau"), - new CodeRangeEntry("pauc", "CR_Pau_Cin_Hau"), - new CodeRangeEntry("oldpermic", "CR_Old_Permic"), - new CodeRangeEntry("perm", "CR_Old_Permic"), - new CodeRangeEntry("psalterpahlavi", "CR_Psalter_Pahlavi"), - new CodeRangeEntry("phlp", "CR_Psalter_Pahlavi"), - new CodeRangeEntry("siddham", "CR_Siddham"), - new CodeRangeEntry("sidd", "CR_Siddham"), - new CodeRangeEntry("khudawadi", "CR_Khudawadi"), - new CodeRangeEntry("sind", "CR_Khudawadi"), - new CodeRangeEntry("tirhuta", "CR_Tirhuta"), - new CodeRangeEntry("tirh", "CR_Tirhuta"), - new CodeRangeEntry("warangciti", "CR_Warang_Citi"), - new CodeRangeEntry("wara", "CR_Warang_Citi"), - new CodeRangeEntry("ahom", "CR_Ahom"), - new CodeRangeEntry("anatolianhieroglyphs", "CR_Anatolian_Hieroglyphs"), - new CodeRangeEntry("hluw", "CR_Anatolian_Hieroglyphs"), - new CodeRangeEntry("hatran", "CR_Hatran"), - new CodeRangeEntry("hatr", "CR_Hatran"), - new CodeRangeEntry("multani", "CR_Multani"), - new CodeRangeEntry("mult", "CR_Multani"), - new CodeRangeEntry("oldhungarian", "CR_Old_Hungarian"), - new CodeRangeEntry("hung", "CR_Old_Hungarian"), - new CodeRangeEntry("signwriting", "CR_SignWriting"), - new CodeRangeEntry("sgnw", "CR_SignWriting"), - new CodeRangeEntry("adlam", "CR_Adlam"), - new CodeRangeEntry("adlm", "CR_Adlam"), - new CodeRangeEntry("bhaiksuki", "CR_Bhaiksuki"), - new CodeRangeEntry("bhks", "CR_Bhaiksuki"), - new CodeRangeEntry("marchen", "CR_Marchen"), - new CodeRangeEntry("marc", "CR_Marchen"), - new CodeRangeEntry("newa", "CR_Newa"), - new CodeRangeEntry("osage", "CR_Osage"), - new CodeRangeEntry("osge", "CR_Osage"), - new CodeRangeEntry("tangut", "CR_Tangut"), - new CodeRangeEntry("tang", "CR_Tangut"), - new CodeRangeEntry("masaramgondi", "CR_Masaram_Gondi"), - new CodeRangeEntry("gonm", "CR_Masaram_Gondi"), - new CodeRangeEntry("nushu", "CR_Nushu"), - new CodeRangeEntry("nshu", "CR_Nushu"), - new CodeRangeEntry("soyombo", "CR_Soyombo"), - new CodeRangeEntry("soyo", "CR_Soyombo"), - new CodeRangeEntry("zanabazarsquare", "CR_Zanabazar_Square"), - new CodeRangeEntry("zanb", "CR_Zanabazar_Square"), - new CodeRangeEntry("whitespace", "CR_Space"), - new CodeRangeEntry("wspace", "CR_Space"), - new CodeRangeEntry("bidicontrol", "CR_Bidi_Control"), - new CodeRangeEntry("bidic", "CR_Bidi_Control"), - new CodeRangeEntry("joincontrol", "CR_Join_Control"), - new CodeRangeEntry("joinc", "CR_Join_Control"), - new CodeRangeEntry("dash", "CR_Dash"), - new CodeRangeEntry("hyphen", "CR_Hyphen"), - new CodeRangeEntry("quotationmark", "CR_Quotation_Mark"), - new CodeRangeEntry("qmark", "CR_Quotation_Mark"), - new CodeRangeEntry("terminalpunctuation", "CR_Terminal_Punctuation"), - new CodeRangeEntry("term", "CR_Terminal_Punctuation"), - new CodeRangeEntry("othermath", "CR_Other_Math"), - new CodeRangeEntry("omath", "CR_Other_Math"), - new CodeRangeEntry("hexdigit", "CR_Hex_Digit"), - new CodeRangeEntry("hex", "CR_Hex_Digit"), - new CodeRangeEntry("asciihexdigit", "CR_XDigit"), - new CodeRangeEntry("ahex", "CR_XDigit"), - new CodeRangeEntry("otheralphabetic", "CR_Other_Alphabetic"), - new CodeRangeEntry("oalpha", "CR_Other_Alphabetic"), - new CodeRangeEntry("ideographic", "CR_Ideographic"), - new CodeRangeEntry("ideo", "CR_Ideographic"), - new CodeRangeEntry("diacritic", "CR_Diacritic"), - new CodeRangeEntry("dia", "CR_Diacritic"), - new CodeRangeEntry("extender", "CR_Extender"), - new CodeRangeEntry("ext", "CR_Extender"), - new CodeRangeEntry("otherlowercase", "CR_Other_Lowercase"), - new CodeRangeEntry("olower", "CR_Other_Lowercase"), - new CodeRangeEntry("otheruppercase", "CR_Other_Uppercase"), - new CodeRangeEntry("oupper", "CR_Other_Uppercase"), - new CodeRangeEntry("noncharactercodepoint", "CR_Noncharacter_Code_Point"), - new CodeRangeEntry("nchar", "CR_Noncharacter_Code_Point"), - new CodeRangeEntry("othergraphemeextend", "CR_Other_Grapheme_Extend"), - new CodeRangeEntry("ogrext", "CR_Other_Grapheme_Extend"), - new CodeRangeEntry("idsbinaryoperator", "CR_IDS_Binary_Operator"), - new CodeRangeEntry("idsb", "CR_IDS_Binary_Operator"), - new CodeRangeEntry("idstrinaryoperator", "CR_IDS_Trinary_Operator"), - new CodeRangeEntry("idst", "CR_IDS_Trinary_Operator"), - new CodeRangeEntry("radical", "CR_Radical"), - new CodeRangeEntry("unifiedideograph", "CR_Unified_Ideograph"), - new CodeRangeEntry("uideo", "CR_Unified_Ideograph"), - new CodeRangeEntry("otherdefaultignorablecodepoint", "CR_Other_Default_Ignorable_Code_Point"), - new CodeRangeEntry("odi", "CR_Other_Default_Ignorable_Code_Point"), - new CodeRangeEntry("deprecated", "CR_Deprecated"), - new CodeRangeEntry("dep", "CR_Deprecated"), - new CodeRangeEntry("softdotted", "CR_Soft_Dotted"), - new CodeRangeEntry("sd", "CR_Soft_Dotted"), - new CodeRangeEntry("logicalorderexception", "CR_Logical_Order_Exception"), - new CodeRangeEntry("loe", "CR_Logical_Order_Exception"), - new CodeRangeEntry("otheridstart", "CR_Other_ID_Start"), - new CodeRangeEntry("oids", "CR_Other_ID_Start"), - new CodeRangeEntry("otheridcontinue", "CR_Other_ID_Continue"), - new CodeRangeEntry("oidc", "CR_Other_ID_Continue"), - new CodeRangeEntry("sentenceterminal", "CR_Sentence_Terminal"), - new CodeRangeEntry("sterm", "CR_Sentence_Terminal"), - new CodeRangeEntry("variationselector", "CR_Variation_Selector"), - new CodeRangeEntry("vs", "CR_Variation_Selector"), - new CodeRangeEntry("patternwhitespace", "CR_Pattern_White_Space"), - new CodeRangeEntry("patws", "CR_Pattern_White_Space"), - new CodeRangeEntry("patternsyntax", "CR_Pattern_Syntax"), - new CodeRangeEntry("patsyn", "CR_Pattern_Syntax"), - new CodeRangeEntry("prependedconcatenationmark", "CR_Prepended_Concatenation_Mark"), - new CodeRangeEntry("pcm", "CR_Prepended_Concatenation_Mark"), - new CodeRangeEntry("graphemeclusterbreak=regionalindicator", "CR_Regional_Indicator"), - new CodeRangeEntry("emoji", "CR_Emoji"), - new CodeRangeEntry("emojipresentation", "CR_Emoji_Presentation"), - new CodeRangeEntry("graphemeclusterbreak=emodifier", "CR_Emoji_Modifier"), - new CodeRangeEntry("emojimodifierbase", "CR_Emoji_Modifier_Base"), - new CodeRangeEntry("emojicomponent", "CR_Emoji_Component"), - new CodeRangeEntry("unknown", "CR_Unknown"), - new CodeRangeEntry("zzzz", "CR_Unknown"), - new CodeRangeEntry("age=1.1", "CR_Age_1_1"), - new CodeRangeEntry("age=2.0", "CR_Age_2_0"), - new CodeRangeEntry("age=2.1", "CR_Age_2_1"), - new CodeRangeEntry("age=3.0", "CR_Age_3_0"), - new CodeRangeEntry("age=3.1", "CR_Age_3_1"), - new CodeRangeEntry("age=3.2", "CR_Age_3_2"), - new CodeRangeEntry("age=4.0", "CR_Age_4_0"), - new CodeRangeEntry("age=4.1", "CR_Age_4_1"), - new CodeRangeEntry("age=5.0", "CR_Age_5_0"), - new CodeRangeEntry("age=5.1", "CR_Age_5_1"), - new CodeRangeEntry("age=5.2", "CR_Age_5_2"), - new CodeRangeEntry("age=6.0", "CR_Age_6_0"), - new CodeRangeEntry("age=6.1", "CR_Age_6_1"), - new CodeRangeEntry("age=6.2", "CR_Age_6_2"), - new CodeRangeEntry("age=6.3", "CR_Age_6_3"), - new CodeRangeEntry("age=7.0", "CR_Age_7_0"), - new CodeRangeEntry("age=8.0", "CR_Age_8_0"), - new CodeRangeEntry("age=9.0", "CR_Age_9_0"), - new CodeRangeEntry("age=10.0", "CR_Age_10_0"), - new CodeRangeEntry("graphemeclusterbreak=prepend", "CR_Grapheme_Cluster_Break_Prepend"), - new CodeRangeEntry("graphemeclusterbreak=cr", "CR_Grapheme_Cluster_Break_CR"), - new CodeRangeEntry("graphemeclusterbreak=lf", "CR_NEWLINE"), - new CodeRangeEntry("graphemeclusterbreak=control", "CR_Grapheme_Cluster_Break_Control"), - new CodeRangeEntry("graphemeextend", "CR_Grapheme_Extend"), - new CodeRangeEntry("grext", "CR_Grapheme_Extend"), - new CodeRangeEntry("regionalindicator", "CR_Regional_Indicator"), - new CodeRangeEntry("ri", "CR_Regional_Indicator"), - new CodeRangeEntry("graphemeclusterbreak=spacingmark", "CR_Grapheme_Cluster_Break_SpacingMark"), - new CodeRangeEntry("graphemeclusterbreak=l", "CR_Grapheme_Cluster_Break_L"), - new CodeRangeEntry("graphemeclusterbreak=v", "CR_Grapheme_Cluster_Break_V"), - new CodeRangeEntry("graphemeclusterbreak=t", "CR_Grapheme_Cluster_Break_T"), - new CodeRangeEntry("graphemeclusterbreak=lv", "CR_Grapheme_Cluster_Break_LV"), - new CodeRangeEntry("graphemeclusterbreak=lvt", "CR_Grapheme_Cluster_Break_LVT"), - new CodeRangeEntry("graphemeclusterbreak=ebase", "CR_Grapheme_Cluster_Break_E_Base"), - new CodeRangeEntry("emojimodifier", "CR_Emoji_Modifier"), - new CodeRangeEntry("graphemeclusterbreak=zwj", "CR_Grapheme_Cluster_Break_ZWJ"), - new CodeRangeEntry("graphemeclusterbreak=glueafterzwj", "CR_Grapheme_Cluster_Break_Glue_After_Zwj"), - new CodeRangeEntry("graphemeclusterbreak=ebasegaz", "CR_Grapheme_Cluster_Break_E_Base_GAZ"), - new CodeRangeEntry("inbasiclatin", "CR_ASCII"), - new CodeRangeEntry("inlatin1supplement", "CR_In_Latin_1_Supplement"), - new CodeRangeEntry("inlatinextendeda", "CR_In_Latin_Extended_A"), - new CodeRangeEntry("inlatinextendedb", "CR_In_Latin_Extended_B"), - new CodeRangeEntry("inipaextensions", "CR_In_IPA_Extensions"), - new CodeRangeEntry("inspacingmodifierletters", "CR_In_Spacing_Modifier_Letters"), - new CodeRangeEntry("incombiningdiacriticalmarks", "CR_In_Combining_Diacritical_Marks"), - new CodeRangeEntry("ingreekandcoptic", "CR_In_Greek_and_Coptic"), - new CodeRangeEntry("incyrillic", "CR_In_Cyrillic"), - new CodeRangeEntry("incyrillicsupplement", "CR_In_Cyrillic_Supplement"), - new CodeRangeEntry("inarmenian", "CR_In_Armenian"), - new CodeRangeEntry("inhebrew", "CR_In_Hebrew"), - new CodeRangeEntry("inarabic", "CR_In_Arabic"), - new CodeRangeEntry("insyriac", "CR_In_Syriac"), - new CodeRangeEntry("inarabicsupplement", "CR_In_Arabic_Supplement"), - new CodeRangeEntry("inthaana", "CR_In_Thaana"), - new CodeRangeEntry("innko", "CR_In_NKo"), - new CodeRangeEntry("insamaritan", "CR_In_Samaritan"), - new CodeRangeEntry("inmandaic", "CR_In_Mandaic"), - new CodeRangeEntry("insyriacsupplement", "CR_In_Syriac_Supplement"), - new CodeRangeEntry("inarabicextendeda", "CR_In_Arabic_Extended_A"), - new CodeRangeEntry("indevanagari", "CR_In_Devanagari"), - new CodeRangeEntry("inbengali", "CR_In_Bengali"), - new CodeRangeEntry("ingurmukhi", "CR_In_Gurmukhi"), - new CodeRangeEntry("ingujarati", "CR_In_Gujarati"), - new CodeRangeEntry("inoriya", "CR_In_Oriya"), - new CodeRangeEntry("intamil", "CR_In_Tamil"), - new CodeRangeEntry("intelugu", "CR_In_Telugu"), - new CodeRangeEntry("inkannada", "CR_In_Kannada"), - new CodeRangeEntry("inmalayalam", "CR_In_Malayalam"), - new CodeRangeEntry("insinhala", "CR_In_Sinhala"), - new CodeRangeEntry("inthai", "CR_In_Thai"), - new CodeRangeEntry("inlao", "CR_In_Lao"), - new CodeRangeEntry("intibetan", "CR_In_Tibetan"), - new CodeRangeEntry("inmyanmar", "CR_In_Myanmar"), - new CodeRangeEntry("ingeorgian", "CR_In_Georgian"), - new CodeRangeEntry("inhanguljamo", "CR_In_Hangul_Jamo"), - new CodeRangeEntry("inethiopic", "CR_In_Ethiopic"), - new CodeRangeEntry("inethiopicsupplement", "CR_In_Ethiopic_Supplement"), - new CodeRangeEntry("incherokee", "CR_In_Cherokee"), - new CodeRangeEntry("inunifiedcanadianaboriginalsyllabics", "CR_In_Unified_Canadian_Aboriginal_Syllabics"), - new CodeRangeEntry("inogham", "CR_In_Ogham"), - new CodeRangeEntry("inrunic", "CR_In_Runic"), - new CodeRangeEntry("intagalog", "CR_In_Tagalog"), - new CodeRangeEntry("inhanunoo", "CR_In_Hanunoo"), - new CodeRangeEntry("inbuhid", "CR_In_Buhid"), - new CodeRangeEntry("intagbanwa", "CR_In_Tagbanwa"), - new CodeRangeEntry("inkhmer", "CR_In_Khmer"), - new CodeRangeEntry("inmongolian", "CR_In_Mongolian"), - new CodeRangeEntry("inunifiedcanadianaboriginalsyllabicsextended", "CR_In_Unified_Canadian_Aboriginal_Syllabics_Extended"), - new CodeRangeEntry("inlimbu", "CR_In_Limbu"), - new CodeRangeEntry("intaile", "CR_In_Tai_Le"), - new CodeRangeEntry("innewtailue", "CR_In_New_Tai_Lue"), - new CodeRangeEntry("inkhmersymbols", "CR_In_Khmer_Symbols"), - new CodeRangeEntry("inbuginese", "CR_In_Buginese"), - new CodeRangeEntry("intaitham", "CR_In_Tai_Tham"), - new CodeRangeEntry("incombiningdiacriticalmarksextended", "CR_In_Combining_Diacritical_Marks_Extended"), - new CodeRangeEntry("inbalinese", "CR_In_Balinese"), - new CodeRangeEntry("insundanese", "CR_In_Sundanese"), - new CodeRangeEntry("inbatak", "CR_In_Batak"), - new CodeRangeEntry("inlepcha", "CR_In_Lepcha"), - new CodeRangeEntry("olchiki", "CR_Ol_Chiki"), - new CodeRangeEntry("olck", "CR_Ol_Chiki"), - new CodeRangeEntry("incyrillicextendedc", "CR_In_Cyrillic_Extended_C"), - new CodeRangeEntry("insundanesesupplement", "CR_In_Sundanese_Supplement"), - new CodeRangeEntry("invedicextensions", "CR_In_Vedic_Extensions"), - new CodeRangeEntry("inphoneticextensions", "CR_In_Phonetic_Extensions"), - new CodeRangeEntry("inphoneticextensionssupplement", "CR_In_Phonetic_Extensions_Supplement"), - new CodeRangeEntry("incombiningdiacriticalmarkssupplement", "CR_In_Combining_Diacritical_Marks_Supplement"), - new CodeRangeEntry("inlatinextendedadditional", "CR_In_Latin_Extended_Additional"), - new CodeRangeEntry("ingreekextended", "CR_In_Greek_Extended"), - new CodeRangeEntry("ingeneralpunctuation", "CR_In_General_Punctuation"), - new CodeRangeEntry("insuperscriptsandsubscripts", "CR_In_Superscripts_and_Subscripts"), - new CodeRangeEntry("incurrencysymbols", "CR_In_Currency_Symbols"), - new CodeRangeEntry("incombiningdiacriticalmarksforsymbols", "CR_In_Combining_Diacritical_Marks_for_Symbols"), - new CodeRangeEntry("inletterlikesymbols", "CR_In_Letterlike_Symbols"), - new CodeRangeEntry("innumberforms", "CR_In_Number_Forms"), - new CodeRangeEntry("inarrows", "CR_In_Arrows"), - new CodeRangeEntry("inmathematicaloperators", "CR_In_Mathematical_Operators"), - new CodeRangeEntry("inmiscellaneoustechnical", "CR_In_Miscellaneous_Technical"), - new CodeRangeEntry("incontrolpictures", "CR_In_Control_Pictures"), - new CodeRangeEntry("inopticalcharacterrecognition", "CR_In_Optical_Character_Recognition"), - new CodeRangeEntry("inenclosedalphanumerics", "CR_In_Enclosed_Alphanumerics"), - new CodeRangeEntry("inboxdrawing", "CR_In_Box_Drawing"), - new CodeRangeEntry("inblockelements", "CR_In_Block_Elements"), - new CodeRangeEntry("ingeometricshapes", "CR_In_Geometric_Shapes"), - new CodeRangeEntry("inmiscellaneoussymbols", "CR_In_Miscellaneous_Symbols"), - new CodeRangeEntry("indingbats", "CR_In_Dingbats"), - new CodeRangeEntry("inmiscellaneousmathematicalsymbolsa", "CR_In_Miscellaneous_Mathematical_Symbols_A"), - new CodeRangeEntry("insupplementalarrowsa", "CR_In_Supplemental_Arrows_A"), - new CodeRangeEntry("braille", "CR_Braille"), - new CodeRangeEntry("brai", "CR_Braille"), - new CodeRangeEntry("insupplementalarrowsb", "CR_In_Supplemental_Arrows_B"), - new CodeRangeEntry("inmiscellaneousmathematicalsymbolsb", "CR_In_Miscellaneous_Mathematical_Symbols_B"), - new CodeRangeEntry("insupplementalmathematicaloperators", "CR_In_Supplemental_Mathematical_Operators"), - new CodeRangeEntry("inmiscellaneoussymbolsandarrows", "CR_In_Miscellaneous_Symbols_and_Arrows"), - new CodeRangeEntry("inglagolitic", "CR_In_Glagolitic"), - new CodeRangeEntry("inlatinextendedc", "CR_In_Latin_Extended_C"), - new CodeRangeEntry("incoptic", "CR_In_Coptic"), - new CodeRangeEntry("ingeorgiansupplement", "CR_In_Georgian_Supplement"), - new CodeRangeEntry("intifinagh", "CR_In_Tifinagh"), - new CodeRangeEntry("inethiopicextended", "CR_In_Ethiopic_Extended"), - new CodeRangeEntry("incyrillicextendeda", "CR_In_Cyrillic_Extended_A"), - new CodeRangeEntry("insupplementalpunctuation", "CR_In_Supplemental_Punctuation"), - new CodeRangeEntry("incjkradicalssupplement", "CR_In_CJK_Radicals_Supplement"), - new CodeRangeEntry("inkangxiradicals", "CR_In_Kangxi_Radicals"), - new CodeRangeEntry("inideographicdescriptioncharacters", "CR_In_Ideographic_Description_Characters"), - new CodeRangeEntry("incjksymbolsandpunctuation", "CR_In_CJK_Symbols_and_Punctuation"), - new CodeRangeEntry("inhiragana", "CR_In_Hiragana"), - new CodeRangeEntry("inkatakana", "CR_In_Katakana"), - new CodeRangeEntry("inbopomofo", "CR_In_Bopomofo"), - new CodeRangeEntry("inhangulcompatibilityjamo", "CR_In_Hangul_Compatibility_Jamo"), - new CodeRangeEntry("inkanbun", "CR_In_Kanbun"), - new CodeRangeEntry("inbopomofoextended", "CR_In_Bopomofo_Extended"), - new CodeRangeEntry("incjkstrokes", "CR_In_CJK_Strokes"), - new CodeRangeEntry("inkatakanaphoneticextensions", "CR_In_Katakana_Phonetic_Extensions"), - new CodeRangeEntry("inenclosedcjklettersandmonths", "CR_In_Enclosed_CJK_Letters_and_Months"), - new CodeRangeEntry("incjkcompatibility", "CR_In_CJK_Compatibility"), - new CodeRangeEntry("incjkunifiedideographsextensiona", "CR_In_CJK_Unified_Ideographs_Extension_A"), - new CodeRangeEntry("inyijinghexagramsymbols", "CR_In_Yijing_Hexagram_Symbols"), - new CodeRangeEntry("incjkunifiedideographs", "CR_In_CJK_Unified_Ideographs"), - new CodeRangeEntry("inyisyllables", "CR_In_Yi_Syllables"), - new CodeRangeEntry("inyiradicals", "CR_In_Yi_Radicals"), - new CodeRangeEntry("lisu", "CR_Lisu"), - new CodeRangeEntry("invai", "CR_In_Vai"), - new CodeRangeEntry("incyrillicextendedb", "CR_In_Cyrillic_Extended_B"), - new CodeRangeEntry("inbamum", "CR_In_Bamum"), - new CodeRangeEntry("inmodifiertoneletters", "CR_In_Modifier_Tone_Letters"), - new CodeRangeEntry("inlatinextendedd", "CR_In_Latin_Extended_D"), - new CodeRangeEntry("insylotinagri", "CR_In_Syloti_Nagri"), - new CodeRangeEntry("incommonindicnumberforms", "CR_In_Common_Indic_Number_Forms"), - new CodeRangeEntry("inphagspa", "CR_In_Phags_pa"), - new CodeRangeEntry("insaurashtra", "CR_In_Saurashtra"), - new CodeRangeEntry("indevanagariextended", "CR_In_Devanagari_Extended"), - new CodeRangeEntry("inkayahli", "CR_In_Kayah_Li"), - new CodeRangeEntry("inrejang", "CR_In_Rejang"), - new CodeRangeEntry("inhanguljamoextendeda", "CR_In_Hangul_Jamo_Extended_A"), - new CodeRangeEntry("injavanese", "CR_In_Javanese"), - new CodeRangeEntry("inmyanmarextendedb", "CR_In_Myanmar_Extended_B"), - new CodeRangeEntry("incham", "CR_In_Cham"), - new CodeRangeEntry("inmyanmarextendeda", "CR_In_Myanmar_Extended_A"), - new CodeRangeEntry("intaiviet", "CR_In_Tai_Viet"), - new CodeRangeEntry("inmeeteimayekextensions", "CR_In_Meetei_Mayek_Extensions"), - new CodeRangeEntry("inethiopicextendeda", "CR_In_Ethiopic_Extended_A"), - new CodeRangeEntry("inlatinextendede", "CR_In_Latin_Extended_E"), - new CodeRangeEntry("incherokeesupplement", "CR_In_Cherokee_Supplement"), - new CodeRangeEntry("inmeeteimayek", "CR_In_Meetei_Mayek"), - new CodeRangeEntry("inhangulsyllables", "CR_In_Hangul_Syllables"), - new CodeRangeEntry("inhanguljamoextendedb", "CR_In_Hangul_Jamo_Extended_B"), - new CodeRangeEntry("inhighsurrogates", "CR_In_High_Surrogates"), - new CodeRangeEntry("inhighprivateusesurrogates", "CR_In_High_Private_Use_Surrogates"), - new CodeRangeEntry("inlowsurrogates", "CR_In_Low_Surrogates"), - new CodeRangeEntry("inprivateusearea", "CR_In_Private_Use_Area"), - new CodeRangeEntry("incjkcompatibilityideographs", "CR_In_CJK_Compatibility_Ideographs"), - new CodeRangeEntry("inalphabeticpresentationforms", "CR_In_Alphabetic_Presentation_Forms"), - new CodeRangeEntry("inarabicpresentationformsa", "CR_In_Arabic_Presentation_Forms_A"), - new CodeRangeEntry("invariationselectors", "CR_In_Variation_Selectors"), - new CodeRangeEntry("inverticalforms", "CR_In_Vertical_Forms"), - new CodeRangeEntry("incombininghalfmarks", "CR_In_Combining_Half_Marks"), - new CodeRangeEntry("incjkcompatibilityforms", "CR_In_CJK_Compatibility_Forms"), - new CodeRangeEntry("insmallformvariants", "CR_In_Small_Form_Variants"), - new CodeRangeEntry("inarabicpresentationformsb", "CR_In_Arabic_Presentation_Forms_B"), - new CodeRangeEntry("inhalfwidthandfullwidthforms", "CR_In_Halfwidth_and_Fullwidth_Forms"), - new CodeRangeEntry("inspecials", "CR_In_Specials"), - new CodeRangeEntry("inlinearbsyllabary", "CR_In_Linear_B_Syllabary"), - new CodeRangeEntry("inlinearbideograms", "CR_In_Linear_B_Ideograms"), - new CodeRangeEntry("inaegeannumbers", "CR_In_Aegean_Numbers"), - new CodeRangeEntry("inancientgreeknumbers", "CR_In_Ancient_Greek_Numbers"), - new CodeRangeEntry("inancientsymbols", "CR_In_Ancient_Symbols"), - new CodeRangeEntry("inphaistosdisc", "CR_In_Phaistos_Disc"), - new CodeRangeEntry("inlycian", "CR_In_Lycian"), - new CodeRangeEntry("incarian", "CR_In_Carian"), - new CodeRangeEntry("incopticepactnumbers", "CR_In_Coptic_Epact_Numbers"), - new CodeRangeEntry("inolditalic", "CR_In_Old_Italic"), - new CodeRangeEntry("ingothic", "CR_In_Gothic"), - new CodeRangeEntry("inoldpermic", "CR_In_Old_Permic"), - new CodeRangeEntry("inugaritic", "CR_In_Ugaritic"), - new CodeRangeEntry("inoldpersian", "CR_In_Old_Persian"), - new CodeRangeEntry("deseret", "CR_Deseret"), - new CodeRangeEntry("dsrt", "CR_Deseret"), - new CodeRangeEntry("shavian", "CR_Shavian"), - new CodeRangeEntry("shaw", "CR_Shavian"), - new CodeRangeEntry("inosmanya", "CR_In_Osmanya"), - new CodeRangeEntry("inosage", "CR_In_Osage"), - new CodeRangeEntry("inelbasan", "CR_In_Elbasan"), - new CodeRangeEntry("incaucasianalbanian", "CR_In_Caucasian_Albanian"), - new CodeRangeEntry("inlineara", "CR_In_Linear_A"), - new CodeRangeEntry("incypriotsyllabary", "CR_In_Cypriot_Syllabary"), - new CodeRangeEntry("inimperialaramaic", "CR_In_Imperial_Aramaic"), - new CodeRangeEntry("palmyrene", "CR_Palmyrene"), - new CodeRangeEntry("palm", "CR_Palmyrene"), - new CodeRangeEntry("innabataean", "CR_In_Nabataean"), - new CodeRangeEntry("inhatran", "CR_In_Hatran"), - new CodeRangeEntry("inphoenician", "CR_In_Phoenician"), - new CodeRangeEntry("inlydian", "CR_In_Lydian"), - new CodeRangeEntry("meroitichieroglyphs", "CR_Meroitic_Hieroglyphs"), - new CodeRangeEntry("mero", "CR_Meroitic_Hieroglyphs"), - new CodeRangeEntry("inmeroiticcursive", "CR_In_Meroitic_Cursive"), - new CodeRangeEntry("inkharoshthi", "CR_In_Kharoshthi"), - new CodeRangeEntry("oldsoutharabian", "CR_Old_South_Arabian"), - new CodeRangeEntry("sarb", "CR_Old_South_Arabian"), - new CodeRangeEntry("oldnortharabian", "CR_Old_North_Arabian"), - new CodeRangeEntry("narb", "CR_Old_North_Arabian"), - new CodeRangeEntry("inmanichaean", "CR_In_Manichaean"), - new CodeRangeEntry("inavestan", "CR_In_Avestan"), - new CodeRangeEntry("ininscriptionalparthian", "CR_In_Inscriptional_Parthian"), - new CodeRangeEntry("ininscriptionalpahlavi", "CR_In_Inscriptional_Pahlavi"), - new CodeRangeEntry("inpsalterpahlavi", "CR_In_Psalter_Pahlavi"), - new CodeRangeEntry("inoldturkic", "CR_In_Old_Turkic"), - new CodeRangeEntry("inoldhungarian", "CR_In_Old_Hungarian"), - new CodeRangeEntry("inruminumeralsymbols", "CR_In_Rumi_Numeral_Symbols"), - new CodeRangeEntry("inbrahmi", "CR_In_Brahmi"), - new CodeRangeEntry("inkaithi", "CR_In_Kaithi"), - new CodeRangeEntry("insorasompeng", "CR_In_Sora_Sompeng"), - new CodeRangeEntry("inchakma", "CR_In_Chakma"), - new CodeRangeEntry("inmahajani", "CR_In_Mahajani"), - new CodeRangeEntry("insharada", "CR_In_Sharada"), - new CodeRangeEntry("insinhalaarchaicnumbers", "CR_In_Sinhala_Archaic_Numbers"), - new CodeRangeEntry("inkhojki", "CR_In_Khojki"), - new CodeRangeEntry("inmultani", "CR_In_Multani"), - new CodeRangeEntry("inkhudawadi", "CR_In_Khudawadi"), - new CodeRangeEntry("ingrantha", "CR_In_Grantha"), - new CodeRangeEntry("innewa", "CR_In_Newa"), - new CodeRangeEntry("intirhuta", "CR_In_Tirhuta"), - new CodeRangeEntry("insiddham", "CR_In_Siddham"), - new CodeRangeEntry("inmodi", "CR_In_Modi"), - new CodeRangeEntry("inmongoliansupplement", "CR_In_Mongolian_Supplement"), - new CodeRangeEntry("intakri", "CR_In_Takri"), - new CodeRangeEntry("inahom", "CR_In_Ahom"), - new CodeRangeEntry("inwarangciti", "CR_In_Warang_Citi"), - new CodeRangeEntry("inzanabazarsquare", "CR_In_Zanabazar_Square"), - new CodeRangeEntry("insoyombo", "CR_In_Soyombo"), - new CodeRangeEntry("inpaucinhau", "CR_In_Pau_Cin_Hau"), - new CodeRangeEntry("inbhaiksuki", "CR_In_Bhaiksuki"), - new CodeRangeEntry("inmarchen", "CR_In_Marchen"), - new CodeRangeEntry("inmasaramgondi", "CR_In_Masaram_Gondi"), - new CodeRangeEntry("incuneiform", "CR_In_Cuneiform"), - new CodeRangeEntry("incuneiformnumbersandpunctuation", "CR_In_Cuneiform_Numbers_and_Punctuation"), - new CodeRangeEntry("inearlydynasticcuneiform", "CR_In_Early_Dynastic_Cuneiform"), - new CodeRangeEntry("inegyptianhieroglyphs", "CR_In_Egyptian_Hieroglyphs"), - new CodeRangeEntry("inanatolianhieroglyphs", "CR_In_Anatolian_Hieroglyphs"), - new CodeRangeEntry("inbamumsupplement", "CR_In_Bamum_Supplement"), - new CodeRangeEntry("inmro", "CR_In_Mro"), - new CodeRangeEntry("inbassavah", "CR_In_Bassa_Vah"), - new CodeRangeEntry("inpahawhhmong", "CR_In_Pahawh_Hmong"), - new CodeRangeEntry("inmiao", "CR_In_Miao"), - new CodeRangeEntry("inideographicsymbolsandpunctuation", "CR_In_Ideographic_Symbols_and_Punctuation"), - new CodeRangeEntry("intangut", "CR_In_Tangut"), - new CodeRangeEntry("intangutcomponents", "CR_In_Tangut_Components"), - new CodeRangeEntry("inkanasupplement", "CR_In_Kana_Supplement"), - new CodeRangeEntry("inkanaextendeda", "CR_In_Kana_Extended_A"), - new CodeRangeEntry("innushu", "CR_In_Nushu"), - new CodeRangeEntry("induployan", "CR_In_Duployan"), - new CodeRangeEntry("inshorthandformatcontrols", "CR_In_Shorthand_Format_Controls"), - new CodeRangeEntry("inbyzantinemusicalsymbols", "CR_In_Byzantine_Musical_Symbols"), - new CodeRangeEntry("inmusicalsymbols", "CR_In_Musical_Symbols"), - new CodeRangeEntry("inancientgreekmusicalnotation", "CR_In_Ancient_Greek_Musical_Notation"), - new CodeRangeEntry("intaixuanjingsymbols", "CR_In_Tai_Xuan_Jing_Symbols"), - new CodeRangeEntry("incountingrodnumerals", "CR_In_Counting_Rod_Numerals"), - new CodeRangeEntry("inmathematicalalphanumericsymbols", "CR_In_Mathematical_Alphanumeric_Symbols"), - new CodeRangeEntry("insuttonsignwriting", "CR_In_Sutton_SignWriting"), - new CodeRangeEntry("inglagoliticsupplement", "CR_In_Glagolitic_Supplement"), - new CodeRangeEntry("inmendekikakui", "CR_In_Mende_Kikakui"), - new CodeRangeEntry("inadlam", "CR_In_Adlam"), - new CodeRangeEntry("inarabicmathematicalalphabeticsymbols", "CR_In_Arabic_Mathematical_Alphabetic_Symbols"), - new CodeRangeEntry("inmahjongtiles", "CR_In_Mahjong_Tiles"), - new CodeRangeEntry("indominotiles", "CR_In_Domino_Tiles"), - new CodeRangeEntry("inplayingcards", "CR_In_Playing_Cards"), - new CodeRangeEntry("inenclosedalphanumericsupplement", "CR_In_Enclosed_Alphanumeric_Supplement"), - new CodeRangeEntry("inenclosedideographicsupplement", "CR_In_Enclosed_Ideographic_Supplement"), - new CodeRangeEntry("inmiscellaneoussymbolsandpictographs", "CR_In_Miscellaneous_Symbols_and_Pictographs"), - new CodeRangeEntry("inemoticons", "CR_In_Emoticons"), - new CodeRangeEntry("inornamentaldingbats", "CR_In_Ornamental_Dingbats"), - new CodeRangeEntry("intransportandmapsymbols", "CR_In_Transport_and_Map_Symbols"), - new CodeRangeEntry("inalchemicalsymbols", "CR_In_Alchemical_Symbols"), - new CodeRangeEntry("ingeometricshapesextended", "CR_In_Geometric_Shapes_Extended"), - new CodeRangeEntry("insupplementalarrowsc", "CR_In_Supplemental_Arrows_C"), - new CodeRangeEntry("insupplementalsymbolsandpictographs", "CR_In_Supplemental_Symbols_and_Pictographs"), - new CodeRangeEntry("incjkunifiedideographsextensionb", "CR_In_CJK_Unified_Ideographs_Extension_B"), - new CodeRangeEntry("incjkunifiedideographsextensionc", "CR_In_CJK_Unified_Ideographs_Extension_C"), - new CodeRangeEntry("incjkunifiedideographsextensiond", "CR_In_CJK_Unified_Ideographs_Extension_D"), - new CodeRangeEntry("incjkunifiedideographsextensione", "CR_In_CJK_Unified_Ideographs_Extension_E"), - new CodeRangeEntry("incjkunifiedideographsextensionf", "CR_In_CJK_Unified_Ideographs_Extension_F"), - new CodeRangeEntry("incjkcompatibilityideographssupplement", "CR_In_CJK_Compatibility_Ideographs_Supplement"), - new CodeRangeEntry("intags", "CR_In_Tags"), - new CodeRangeEntry("invariationselectorssupplement", "CR_In_Variation_Selectors_Supplement"), - new CodeRangeEntry("insupplementaryprivateuseareaa", "CR_In_Supplementary_Private_Use_Area_A"), - new CodeRangeEntry("insupplementaryprivateuseareab", "CR_In_Supplementary_Private_Use_Area_B"), - new CodeRangeEntry("innoblock", "CR_In_No_Block") - }; - - static final int MAX_WORD_LENGTH = 44; -} diff --git a/src/org/jcodings/util/ArrayReader.java b/src/org/jcodings/util/ArrayReader.java index 5050c253..e07822e4 100644 --- a/src/org/jcodings/util/ArrayReader.java +++ b/src/org/jcodings/util/ArrayReader.java @@ -44,12 +44,17 @@ public static byte[] readByteArray(String name) { bytes[i] = dis.readByte(); } checkAvailable(dis, name); - dis.close(); return bytes; } catch (IOException ioe) { decorate(ioe, name); - return null; + } finally { + try { + dis.close(); + } catch (IOException ex) { + // ignore + } } + return null; } public static int[] readIntArray(String name) { @@ -61,12 +66,17 @@ public static int[] readIntArray(String name) { ints[i] = dis.readInt(); } checkAvailable(dis, name); - dis.close(); return ints; } catch (IOException ioe) { decorate(ioe, name); - return null; - } + } finally { + try { + dis.close(); + } catch (IOException ex) { + // ignore + } + } + return null; } public static int[][] readNestedIntArray(String name) { @@ -82,14 +92,18 @@ public static int[][] readNestedIntArray(String name) { iints[k] = dis.readInt(); } } - checkAvailable(dis, name); - dis.close(); return ints; } catch (IOException ioe) { decorate(ioe, name); - return null; + } finally { + try { + dis.close(); + } catch (IOException ex) { + // ignore + } } + return null; } static void checkAvailable(DataInputStream dis, String name) throws IOException { diff --git a/src/org/jcodings/util/BytesHash.java b/src/org/jcodings/util/BytesHash.java index 3aeb11cf..7237735a 100644 --- a/src/org/jcodings/util/BytesHash.java +++ b/src/org/jcodings/util/BytesHash.java @@ -29,6 +29,7 @@ public BytesHash(int size) { super(size); } + @Override protected void init() { head = new BytesHashEntry(); } @@ -62,7 +63,7 @@ public boolean equals(byte[]bytes, int p, int end) { public static int hashCode(byte[]bytes, int p, int end) { int key = 0; - while (p < end) key = ((key << 16) + (key << 6) - key) + (int)(bytes[p++]); // & 0xff ? we have to match jruby string hash + while (p < end) key = ((key << 16) + (key << 6) - key) + bytes[p++]; // & 0xff ? we have to match jruby string hash key = key + (key >> 5); return key; } diff --git a/src/org/jcodings/util/CaseInsensitiveBytesHash.java b/src/org/jcodings/util/CaseInsensitiveBytesHash.java index 6f30eb89..248ef39b 100644 --- a/src/org/jcodings/util/CaseInsensitiveBytesHash.java +++ b/src/org/jcodings/util/CaseInsensitiveBytesHash.java @@ -31,6 +31,7 @@ public CaseInsensitiveBytesHash(int size) { super(size); } + @Override protected void init() { head = new CaseInsensitiveBytesHashEntry(); } @@ -60,7 +61,7 @@ public boolean equals(byte[]bytes, int p, int end) { public static int hashCode(byte[]bytes, int p, int end) { int key = 0; - while (p < end) key = ((key << 16) + (key << 6) - key) + (int)(ToLowerCaseTable[bytes[p++] & 0xff]); // & 0xff ? we have to match jruby string hash + while (p < end) key = ((key << 16) + (key << 6) - key) + ToLowerCaseTable[bytes[p++] & 0xff]; // & 0xff ? we have to match jruby string hash key = key + (key >> 5); return key; } diff --git a/src/org/jcodings/util/Hash.java b/src/org/jcodings/util/Hash.java index 227aa0ea..e973018f 100644 --- a/src/org/jcodings/util/Hash.java +++ b/src/org/jcodings/util/Hash.java @@ -133,6 +133,7 @@ protected static int hashValue(int h) { return h & HASH_SIGN_BIT_MASK; } + @Override public Iterator iterator() { return new HashIterator(); } @@ -144,16 +145,19 @@ public HashIterator() { next = head.after; } + @Override public boolean hasNext() { return next != head; } + @Override public V next() { HashEntry e = next; next = e.after; return e.value; } + @Override public void remove() { throw new InternalException("not supported operation exception"); } @@ -170,20 +174,24 @@ public HashEntryIterator() { next = head.after; } + @Override public Iterator> iterator() { return this; } + @Override public boolean hasNext() { return next != head; } + @Override public HashEntry next() { HashEntry e = next; next = e.after; return e; } + @Override public void remove() { throw new InternalException("not supported operation exception"); } diff --git a/src/org/jcodings/util/IntArrayHash.java b/src/org/jcodings/util/IntArrayHash.java index 22e1f5f6..e9b588bf 100644 --- a/src/org/jcodings/util/IntArrayHash.java +++ b/src/org/jcodings/util/IntArrayHash.java @@ -29,6 +29,7 @@ public IntArrayHash(int size) { super(size); } + @Override protected void init() { head = new IntArrayHashEntry(); } diff --git a/src/org/jcodings/util/IntHash.java b/src/org/jcodings/util/IntHash.java index 9b0c1eba..20d6da7f 100644 --- a/src/org/jcodings/util/IntHash.java +++ b/src/org/jcodings/util/IntHash.java @@ -29,6 +29,7 @@ public IntHash(int size) { super(size); } + @Override protected void init() { head = new IntHashEntry(); } diff --git a/src/org/jcodings/util/Macros.java b/src/org/jcodings/util/Macros.java index c8ed0910..829c208b 100644 --- a/src/org/jcodings/util/Macros.java +++ b/src/org/jcodings/util/Macros.java @@ -48,7 +48,7 @@ public static int CONSTRUCT_MBCLEN_CHARFOUND(int n) { // UNICODE_VALID_CODEPOINT_P public static boolean UNICODE_VALID_CODEPOINT_P(int c) { - return ((c) <= 0x10ffff) && + return (Integer.compareUnsigned(c, 0x10ffff) <= 0) && !((c) < 0x10000 && UTF16_IS_SURROGATE((c) >> 8)); } diff --git a/src/org/jcodings/util/ObjHash.java b/src/org/jcodings/util/ObjHash.java index 0403d0a0..8b96020f 100644 --- a/src/org/jcodings/util/ObjHash.java +++ b/src/org/jcodings/util/ObjHash.java @@ -21,6 +21,7 @@ public final class ObjHash extends Hash { + @Override protected void init() { head = new ObjHashEntry(); } @@ -38,6 +39,7 @@ public ObjHashEntry() { key = null; } + @Override public boolean equals(Object key) { if (this.key == key) return true; return this.key.equals(key); diff --git a/test/org/jcodings/specific/TestBig5.java b/test/org/jcodings/specific/TestBig5.java index fdba5f1a..7d5a10bb 100755 --- a/test/org/jcodings/specific/TestBig5.java +++ b/test/org/jcodings/specific/TestBig5.java @@ -5,7 +5,6 @@ import java.nio.charset.Charset; -import org.jcodings.specific.BIG5Encoding; import org.junit.Test; public class TestBig5 { diff --git a/test/org/jcodings/specific/TestCESU8.java b/test/org/jcodings/specific/TestCESU8.java new file mode 100644 index 00000000..62e7fe81 --- /dev/null +++ b/test/org/jcodings/specific/TestCESU8.java @@ -0,0 +1,52 @@ +package org.jcodings.specific; + +import static org.junit.Assert.assertEquals; + +import org.jcodings.Encoding; +import org.junit.Test; + +public class TestCESU8 { + final Encoding enc = CESU8Encoding.INSTANCE; + + @Test + public void testUnicodeLength6byteChar() throws Exception { + byte[]bytes = "\u00ed\u00a0\u0080\u00ed\u00b0\u0080".getBytes("ISO-8859-1"); + assertEquals(1, enc.strLength(bytes, 0, bytes.length)); + } + + @Test + public void testPrevCharHead6byteChar() throws Exception { + byte[]bytes = "\u00ed\u00a0\u0080\u00ed\u00b0\u0080".getBytes("ISO-8859-1"); + assertEquals(0, enc.prevCharHead(bytes, 0, bytes.length, bytes.length)); + } + + @Test + public void testUnicodeLength() throws Exception { + byte[]bytes = "test\u00C5\u0099".getBytes(); + assertEquals(6, enc.strLength(bytes, 0, bytes.length)); + } + + @Test + public void testUnicodeLengthLong() throws Exception { + byte[]bytes = ("\u00C5\u0099\u00C5\u00A1\u00C4\u009B\u00C5\u0099\u00C5\u00A1\u00C4\u009B\u00C5\u0099\u00C3\u00A9\u00C4" + + "\u009B\u00C3\u00BD\u00C5\u0099\u00C5\u00A1\u00C4\u009B\u00C3\u00A9\u00C4\u009B\u00C3\u00A9\u00C5\u00BE\u00C4\u009B\u00C5\u00A1" + + "\u00C3\u00A9\u00C5\u00BE\u00C4\u009B\u00C5\u00BE\u00C3\u00A9\u00C4\u009B\u00C5\u00A1").getBytes("ISO-8859-1"); + assertEquals(26, enc.strLength(bytes, 0, bytes.length)); + } + + @Test + public void testCodeToMbcLength() throws Exception { + assertEquals(enc.codeToMbcLength(0x01), 1); + assertEquals(enc.codeToMbcLength(0x1F608), 6); + } + + @Test + public void testMbcToCode() throws Exception { + assertEquals('Ø', enc.mbcToCode("mØØse".getBytes("UTF-8"), 1, 3)); + } + + @Test + public void testEncodingLoad() throws Exception { + assertEquals(CESU8Encoding.INSTANCE, Encoding.load("CESU8")); + } +} diff --git a/test/org/jcodings/specific/TestCaseMap.java b/test/org/jcodings/specific/TestCaseMap.java index d79ad014..9ad7e5df 100644 --- a/test/org/jcodings/specific/TestCaseMap.java +++ b/test/org/jcodings/specific/TestCaseMap.java @@ -1,6 +1,6 @@ package org.jcodings.specific; -import static junit.framework.Assert.assertTrue; +import static org.junit.Assert.assertTrue; import java.util.Arrays; import java.util.List; @@ -81,4 +81,17 @@ public void testISOCaseMap() throws Exception { assertTrue(caseMap(enc, "ß", Config.CASE_DOWNCASE).equals("ß")); } } + + @Test + public void testGeorgian() throws Exception { + Encoding enc = UTF8Encoding.INSTANCE; + assertTrue(caseMap(enc, "\u1C90\u1C91\u1C92", Config.CASE_UPCASE | Config.CASE_TITLECASE).equals("\u10D0\u10D1\u10D2")); + assertTrue(caseMap(enc, "\u1C90\u1C91\u10D2", Config.CASE_UPCASE | Config.CASE_TITLECASE).equals("\u10D0\u10D1\u10D2")); + assertTrue(caseMap(enc, "\u1C90\u10D1\u1C92", Config.CASE_UPCASE | Config.CASE_TITLECASE).equals("\u10D0\u10D1\u10D2")); + assertTrue(caseMap(enc, "\u1C90\u10D1\u10D2", Config.CASE_UPCASE | Config.CASE_TITLECASE).equals("\u10D0\u10D1\u10D2")); + assertTrue(caseMap(enc, "\u10D0\u1C91\u1C92", Config.CASE_UPCASE | Config.CASE_TITLECASE).equals("\u10D0\u10D1\u10D2")); + assertTrue(caseMap(enc, "\u10D0\u1C91\u10D2", Config.CASE_UPCASE | Config.CASE_TITLECASE).equals("\u10D0\u10D1\u10D2")); + assertTrue(caseMap(enc, "\u10D0\u10D1\u1C92", Config.CASE_UPCASE | Config.CASE_TITLECASE).equals("\u10D0\u10D1\u10D2")); + assertTrue(caseMap(enc, "\u10D0\u10D1\u10D2", Config.CASE_UPCASE | Config.CASE_TITLECASE).equals("\u10D0\u10D1\u10D2")); + } } diff --git a/test/org/jcodings/specific/TestEConv.java b/test/org/jcodings/specific/TestEConv.java index 326c73f3..69f3bfe7 100644 --- a/test/org/jcodings/specific/TestEConv.java +++ b/test/org/jcodings/specific/TestEConv.java @@ -63,6 +63,18 @@ public void testCrNewline() throws Exception { Assert.assertArrayEquals("foo\rbar".getBytes(), dest); } + @Test + public void testLfNewline() throws Exception { + EConv econv = TranscoderDB.open("".getBytes(), "".getBytes(), EConvFlags.LF_NEWLINE_DECORATOR); + + byte[] src = "foo\nbar\r\nbaz".getBytes(); + byte[] dest = new byte[11]; + + econv.convert(src, new Ptr(0), 12, dest, new Ptr(0), dest.length, 0); + + Assert.assertArrayEquals("foo\nbar\nbaz".getBytes(), dest); + } + @Test public void testXMLWithCharref() throws Exception { EConv econv = TranscoderDB.open("utf-8".getBytes(), "euc-jp".getBytes(), EConvFlags.XML_ATTR_CONTENT_DECORATOR | EConvFlags.XML_ATTR_QUOTE_DECORATOR | EConvFlags.UNDEF_HEX_CHARREF); diff --git a/test/org/jcodings/specific/TestEConvResult.java b/test/org/jcodings/specific/TestEConvResult.java index dc02fdbf..5676749e 100644 --- a/test/org/jcodings/specific/TestEConvResult.java +++ b/test/org/jcodings/specific/TestEConvResult.java @@ -1,14 +1,15 @@ package org.jcodings.specific; -import junit.framework.Assert; +import static org.junit.Assert.assertEquals; + import org.jcodings.transcode.EConvResult; import org.junit.Test; public class TestEConvResult { @Test public void testSymbolicName() { - Assert.assertEquals("finished", EConvResult.Finished.symbolicName()); - Assert.assertEquals("after_output", EConvResult.AfterOutput.symbolicName()); - Assert.assertEquals("destination_buffer_full", EConvResult.DestinationBufferFull.symbolicName()); + assertEquals("finished", EConvResult.Finished.symbolicName()); + assertEquals("after_output", EConvResult.AfterOutput.symbolicName()); + assertEquals("destination_buffer_full", EConvResult.DestinationBufferFull.symbolicName()); } } diff --git a/test/org/jcodings/specific/TestEUCJP.java b/test/org/jcodings/specific/TestEUCJP.java index 5624122b..915f3d21 100755 --- a/test/org/jcodings/specific/TestEUCJP.java +++ b/test/org/jcodings/specific/TestEUCJP.java @@ -1,11 +1,12 @@ package org.jcodings.specific; +import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assume.assumeTrue; import java.nio.charset.Charset; -import org.jcodings.specific.EUCJPEncoding; +import org.jcodings.IntHolder; import org.junit.Test; public class TestEUCJP { @@ -24,4 +25,21 @@ public void testGetCharsetName() { "EUC-JP", EUCJPEncoding.INSTANCE.getCharsetName()); } + + @Test + public void testCaseFold() { + EUCJPEncoding enc = EUCJPEncoding.INSTANCE; + byte [] lowerSrc = new byte[]{(byte)0xA3, (byte)0xE1}; + byte [] upperSrc = new byte[]{(byte)0xA3, (byte)0xC1}; + byte [] lower = new byte[2]; + IntHolder pp = new IntHolder(); + + pp.value = 0; + enc.mbcCaseFold(0, lowerSrc, pp, 2, lower); + assertArrayEquals(lowerSrc, lower); + + pp.value = 0; + enc.mbcCaseFold(0, upperSrc, pp, 2, lower); + assertArrayEquals(lowerSrc, lower); + } } diff --git a/test/org/jcodings/specific/TestEmacsMule.java b/test/org/jcodings/specific/TestEmacsMule.java index 9bd983d6..68253abc 100644 --- a/test/org/jcodings/specific/TestEmacsMule.java +++ b/test/org/jcodings/specific/TestEmacsMule.java @@ -1,14 +1,13 @@ package org.jcodings.specific; import static org.junit.Assert.assertEquals; -import org.jcodings.specific.EmacsMuleEncoding; import org.junit.Test; public class TestEmacsMule { @Test public void testRightAdjustCharHeadAscii() { byte[] str = new byte[]{(byte)'a', (byte)'b', (byte)'c', (byte)',', (byte)'d', (byte)'e', (byte)'f'}; - + int t = EmacsMuleEncoding.INSTANCE.rightAdjustCharHead(str, 0, 3, 7); assertEquals("rightAdjustCharHead did not adjust properly", 3, t); } diff --git a/test/org/jcodings/specific/TestGBK.java b/test/org/jcodings/specific/TestGBK.java index 03a6f334..fa541dfb 100755 --- a/test/org/jcodings/specific/TestGBK.java +++ b/test/org/jcodings/specific/TestGBK.java @@ -8,7 +8,6 @@ import org.jcodings.Encoding; import org.jcodings.EncodingDB; import org.jcodings.EncodingDB.Entry; -import org.jcodings.specific.GBKEncoding; import org.jcodings.util.CaseInsensitiveBytesHash; import org.junit.Before; import org.junit.Test; diff --git a/test/org/jcodings/specific/TestSJIS.java b/test/org/jcodings/specific/TestSJIS.java new file mode 100644 index 00000000..06c635e1 --- /dev/null +++ b/test/org/jcodings/specific/TestSJIS.java @@ -0,0 +1,26 @@ +package org.jcodings.specific; + +import static org.junit.Assert.assertArrayEquals; + +import org.jcodings.IntHolder; +import org.junit.Test; + +public class TestSJIS { + + @Test + public void testCaseFold() { + SJISEncoding enc = SJISEncoding.INSTANCE; + byte [] lowerSrc = new byte[]{(byte)0x82, (byte)0x81}; + byte [] upperSrc = new byte[]{(byte)0x82, (byte)0x60}; + byte [] lower = new byte[2]; + IntHolder pp = new IntHolder(); + + pp.value = 0; + enc.mbcCaseFold(0, lowerSrc, pp, 2, lower); + assertArrayEquals(lowerSrc, lower); + + pp.value = 0; + enc.mbcCaseFold(0, upperSrc, pp, 2, lower); + assertArrayEquals(lowerSrc, lower); + } +} \ No newline at end of file diff --git a/test/org/jcodings/specific/TestUnicode.java b/test/org/jcodings/specific/TestUnicode.java index 5eae407a..08a87840 100644 --- a/test/org/jcodings/specific/TestUnicode.java +++ b/test/org/jcodings/specific/TestUnicode.java @@ -1,7 +1,7 @@ package org.jcodings.specific; -import static junit.framework.Assert.assertEquals; -import static junit.framework.Assert.assertTrue; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; import org.jcodings.Encoding; import org.jcodings.constants.CharacterType; @@ -45,4 +45,9 @@ public void testCodeToMbcLength() throws Exception { assertEquals(enc.codeToMbcLength(0xfffffffe), 1); // USE_INVALID_CODE_SCHEME assertEquals(enc.codeToMbcLength(0xffffffff), 1); // USE_INVALID_CODE_SCHEME } + + @Test + public void testEncodingLoad() throws Exception { + assertEquals(UTF8Encoding.INSTANCE, Encoding.load("UTF8")); + } } diff --git a/test/org/jcodings/transcode/TestCP51932ToCP50220.java b/test/org/jcodings/transcode/TestCP51932ToCP50220.java new file mode 100644 index 00000000..d1cf8b4d --- /dev/null +++ b/test/org/jcodings/transcode/TestCP51932ToCP50220.java @@ -0,0 +1,24 @@ +package org.jcodings.transcode; + +import org.jcodings.Ptr; +import org.junit.Test; +import org.junit.Assert; +import java.util.Arrays; + +public class TestCP51932ToCP50220 { + @Test + public void testCP51932ToCP50220() throws Exception { + byte[] src = "\u008E\u00A1\u008E\u00FE".getBytes("iso-8859-1"); + byte[] dst = new byte[100]; + Ptr srcPtr = new Ptr(0); + Ptr dstPtr = new Ptr(0); + EConv econv = TranscoderDB.open("CP51932", "CP50220", 0); + econv.convert(src, srcPtr, src.length, dst, dstPtr, dst.length, 0); + + byte[] str = Arrays.copyOf(dst, dstPtr.p); + + byte[] expected = "\u001B\u0024\u0042\u0021\u0023\u001B\u0028\u0049\u007E\u001B\u0028\u0042".getBytes("iso-8859-1"); + byte[] actual = Arrays.copyOf(dst, dstPtr.p); + Assert.assertEquals(new String(expected), new String(actual)); + } +} \ No newline at end of file