This is an automated email from the ASF dual-hosted git repository. tallison pushed a commit to branch TIKA-4692-improve-ooxml-sax-parsers in repository https://gitbox.apache.org/repos/asf/tika.git
commit 86e2133d768468a5e6ddb3b01bfaed35f6a62e2f Merge: 8eec3f2999 8dc2999ae0 Author: tallison <[email protected]> AuthorDate: Fri Apr 3 07:00:11 2026 -0400 Merge remote-tracking branch 'origin/main' into TIKA-4692-improve-ooxml-sax-parsers .mvn/extensions.xml | 2 +- CHANGES.txt | 4 + .../ROOT/pages/advanced/charset-detection-eval.txt | 274 + .../advanced/charsoup-supported-languages.adoc | 207 +- .../ROOT/pages/advanced/flores-eval-20260320.txt | 580 + .../pages/advanced/generative-language-model.adoc | 140 +- .../pages/advanced/language-detection-build.adoc | 621 +- .../ROOT/pages/advanced/language-detection.adoc | 2 +- tika-bom/pom.xml | 61 + .../main/java/org/apache/tika/metadata/MAPI.java | 14 + .../main/java/org/apache/tika/metadata/Office.java | 4 + .../org/apache/tika/sax/XHTMLContentHandler.java | 1 - .../main/java/org/apache/tika/utils/DateUtils.java | 3 + .../org/apache/tika/mime/tika-mimetypes.xml | 6 +- .../resources/pipes-fork-server-default-log4j2.xml | 2 +- tika-e2e-tests/pom.xml | 3 +- tika-e2e-tests/tika-server/pom.xml | 139 + .../tika/server/e2e/TikaServerHttp2Test.java | 217 + .../charsoup/CharSoupEncodingDetector.java | 128 +- .../org/apache/tika/eval/app/ExtractComparer.java | 57 +- .../src/main/resources/comparison-reports-tags.xml | 25 + .../src/main/resources/comparison-reports.xml | 26 + .../eval/core/metadata/TikaEvalMetadataFilter.java | 10 +- .../src/main/resources/common_tokens/ace | 3261 +- .../src/main/resources/common_tokens/afr | 58582 +++++++++--------- .../src/main/resources/common_tokens/aka | 19339 +++--- .../src/main/resources/common_tokens/alt | 5673 +- .../src/main/resources/common_tokens/amh | 19160 +++--- .../src/main/resources/common_tokens/ami | 5460 +- .../src/main/resources/common_tokens/ara | 59812 +++++++++--------- .../src/main/resources/common_tokens/arg | 48885 ++++++++------- .../src/main/resources/common_tokens/asm | 10416 ++-- .../src/main/resources/common_tokens/ava | 5305 +- .../src/main/resources/common_tokens/avk | 14236 +++-- .../src/main/resources/common_tokens/azb | 48707 ++++++++------- .../src/main/resources/common_tokens/aze | 59620 +++++++++--------- .../src/main/resources/common_tokens/bak | 59078 +++++++++--------- .../src/main/resources/common_tokens/ban | 14234 +++-- .../src/main/resources/common_tokens/bar | 31353 +++++----- .../src/main/resources/common_tokens/bcl | 24806 ++++---- .../src/main/resources/common_tokens/be-x-old | 59662 +++++++++--------- .../src/main/resources/common_tokens/bel | 59732 +++++++++--------- .../src/main/resources/common_tokens/ben | 28389 +++++---- .../src/main/resources/common_tokens/bjn | 10079 ++-- .../src/main/resources/common_tokens/bod | 527 + .../src/main/resources/common_tokens/bre | 56480 ++++++++--------- .../src/main/resources/common_tokens/bul | 59778 +++++++++--------- .../src/main/resources/common_tokens/bxr | 7601 +-- .../src/main/resources/common_tokens/cat | 59066 +++++++++--------- .../src/main/resources/common_tokens/cdo-x-rom | 1279 +- .../src/main/resources/common_tokens/ceb | 53368 ++++++++--------- .../src/main/resources/common_tokens/ces | 59868 +++++++++--------- .../src/main/resources/common_tokens/che | 56598 ++++++++--------- .../src/main/resources/common_tokens/chv | 33025 +++++----- .../src/main/resources/common_tokens/ckb | 58370 +++++++++--------- .../src/main/resources/common_tokens/cnh | 35018 ++++++----- .../src/main/resources/common_tokens/cor | 7559 +-- .../src/main/resources/common_tokens/cos | 16434 ++--- .../src/main/resources/common_tokens/csb | 4494 +- .../src/main/resources/common_tokens/cym | 57934 +++++++++--------- .../src/main/resources/common_tokens/dag | 9356 +-- .../src/main/resources/common_tokens/dan | 59036 +++++++++--------- .../src/main/resources/common_tokens/deu | 59112 +++++++++--------- .../src/main/resources/common_tokens/diq | 35052 +++++++++-- .../src/main/resources/common_tokens/div | 15474 ++++- .../src/main/resources/common_tokens/dsb | 5063 +- .../src/main/resources/common_tokens/ell | 59866 +++++++++--------- .../src/main/resources/common_tokens/eng | 58222 +++++++++--------- .../src/main/resources/common_tokens/epo | 58852 +++++++++--------- .../src/main/resources/common_tokens/est | 59338 +++++++++--------- .../src/main/resources/common_tokens/eus | 59158 +++++++++--------- .../src/main/resources/common_tokens/ewe | 22320 +++---- .../src/main/resources/common_tokens/ext | 6421 +- .../src/main/resources/common_tokens/fao | 20477 ++++--- .../src/main/resources/common_tokens/fas | 57870 +++++++++--------- .../src/main/resources/common_tokens/fin | 59528 +++++++++--------- .../src/main/resources/common_tokens/fra | 58382 +++++++++--------- .../src/main/resources/common_tokens/frr | 10142 ++-- .../src/main/resources/common_tokens/fry | 59206 +++++++++--------- .../src/main/resources/common_tokens/gla | 14147 +++-- .../src/main/resources/common_tokens/gle | 52606 ++++++++-------- .../src/main/resources/common_tokens/glg | 59292 +++++++++--------- .../src/main/resources/common_tokens/glv | 7881 +-- .../src/main/resources/common_tokens/gom | 10524 ++-- .../src/main/resources/common_tokens/grn | 9707 +-- .../src/main/resources/common_tokens/gsw | 59314 +++++++++--------- .../src/main/resources/common_tokens/guj | 11744 ++-- .../src/main/resources/common_tokens/hak-x-rom | 1421 +- .../src/main/resources/common_tokens/hau | 45690 +++++++------- .../src/main/resources/common_tokens/heb | 59994 +++++++++--------- .../src/main/resources/common_tokens/hil | 38597 ++++++------ .../src/main/resources/common_tokens/hin | 16128 +++-- .../src/main/resources/common_tokens/hrv | 59876 +++++++++--------- .../src/main/resources/common_tokens/hsb | 17857 +++--- .../src/main/resources/common_tokens/hun | 59516 +++++++++--------- .../src/main/resources/common_tokens/hye | 59522 +++++++++--------- .../src/main/resources/common_tokens/hyw | 40638 +++++++------ .../src/main/resources/common_tokens/ibo | 36791 ++++++------ .../src/main/resources/common_tokens/ido | 28746 +++++---- .../src/main/resources/common_tokens/ile | 6617 +- .../src/main/resources/common_tokens/ilo | 13590 +++-- .../src/main/resources/common_tokens/ina | 19257 +++--- .../src/main/resources/common_tokens/ind | 57836 +++++++++--------- .../src/main/resources/common_tokens/isl | 59426 +++++++++--------- .../src/main/resources/common_tokens/ita | 59098 +++++++++--------- .../src/main/resources/common_tokens/jav | 58619 +++++++++--------- .../src/main/resources/common_tokens/jbo | 2284 +- .../src/main/resources/common_tokens/jpn | 59914 +++++++++--------- .../src/main/resources/common_tokens/kaa | 9998 +-- .../src/main/resources/common_tokens/kab | 6933 ++- .../src/main/resources/common_tokens/kan | 57362 +++++++++--------- .../src/main/resources/common_tokens/kat | 59766 +++++++++--------- .../src/main/resources/common_tokens/kaz | 59410 +++++++++--------- .../src/main/resources/common_tokens/kha | 20048 ++++--- .../src/main/resources/common_tokens/khm | 19506 +++--- .../src/main/resources/common_tokens/kin | 16652 ++--- .../src/main/resources/common_tokens/kir | 59714 +++++++++--------- .../src/main/resources/common_tokens/kor | 59970 +++++++++--------- .../src/main/resources/common_tokens/kpv | 7300 ++- .../src/main/resources/common_tokens/ksh | 5224 +- .../src/main/resources/common_tokens/kur | 41528 +++++++------ .../src/main/resources/common_tokens/lao | 1670 +- .../src/main/resources/common_tokens/lat | 58786 +++++++++--------- .../src/main/resources/common_tokens/lav | 59622 +++++++++--------- .../src/main/resources/common_tokens/lez | 8326 +-- .../src/main/resources/common_tokens/lfn | 11797 ++-- .../src/main/resources/common_tokens/lim | 40637 +++++++------ .../src/main/resources/common_tokens/lit | 59478 +++++++++--------- .../src/main/resources/common_tokens/ltz | 57304 +++++++++--------- .../src/main/resources/common_tokens/lug | 10940 ++-- .../src/main/resources/common_tokens/lus | 55276 ++++++++--------- .../src/main/resources/common_tokens/mal | 57366 +++++++++--------- .../src/main/resources/common_tokens/mar | 20721 ++++--- .../src/main/resources/common_tokens/mhr | 11844 ++-- .../src/main/resources/common_tokens/min | 33269 +++++----- .../src/main/resources/common_tokens/mkd | 59796 +++++++++--------- .../src/main/resources/common_tokens/mlg | 55190 ++++++++--------- .../src/main/resources/common_tokens/mlt | 37519 ++++++------ .../src/main/resources/common_tokens/mon | 57267 +++++++++--------- .../src/main/resources/common_tokens/mrj | 5384 +- .../src/main/resources/common_tokens/msa | 56960 +++++++++--------- .../src/main/resources/common_tokens/mwl | 30033 +++++----- .../src/main/resources/common_tokens/mya | 59952 +++++++++--------- .../src/main/resources/common_tokens/myv | 4626 +- .../src/main/resources/common_tokens/mzn | 10865 ++-- .../src/main/resources/common_tokens/nds | 55806 ++++++++--------- .../src/main/resources/common_tokens/nep | 12508 ++-- .../src/main/resources/common_tokens/nld | 56386 ++++++++--------- .../src/main/resources/common_tokens/nno | 58638 +++++++++--------- .../src/main/resources/common_tokens/nob | 58518 +++++++++--------- .../src/main/resources/common_tokens/nqo | 5836 +- .../src/main/resources/common_tokens/nso | 2416 +- .../src/main/resources/common_tokens/nya | 57552 +++++++++--------- .../src/main/resources/common_tokens/olo | 4824 +- .../src/main/resources/common_tokens/ori | 11158 ++-- .../src/main/resources/common_tokens/orm | 58182 +++++++++--------- .../src/main/resources/common_tokens/oss | 9637 +-- .../src/main/resources/common_tokens/pam | 7385 ++- .../src/main/resources/common_tokens/pan | 6048 +- .../src/main/resources/common_tokens/pap | 7386 +-- .../src/main/resources/common_tokens/pfl | 4325 +- .../src/main/resources/common_tokens/pnb | 57824 +++++++++--------- .../src/main/resources/common_tokens/pol | 59670 +++++++++--------- .../src/main/resources/common_tokens/por | 59182 +++++++++--------- .../src/main/resources/common_tokens/pus | 59644 +++++++++--------- .../src/main/resources/common_tokens/roh | 25077 ++++---- .../src/main/resources/common_tokens/ron | 58502 +++++++++--------- .../src/main/resources/common_tokens/rue | 8460 +-- .../src/main/resources/common_tokens/rus | 59874 +++++++++--------- .../src/main/resources/common_tokens/sah | 37487 ++++++------ .../src/main/resources/common_tokens/san | 11790 ++-- .../src/main/resources/common_tokens/sat | 13787 +++-- .../src/main/resources/common_tokens/sgs | 6831 ++- .../src/main/resources/common_tokens/sin | 31996 +++++----- .../src/main/resources/common_tokens/skr | 13330 ++-- .../src/main/resources/common_tokens/slk | 59810 +++++++++--------- .../src/main/resources/common_tokens/slv | 59774 +++++++++--------- .../src/main/resources/common_tokens/sme | 3991 +- .../src/main/resources/common_tokens/smn | 6356 +- .../src/main/resources/common_tokens/smo | 42362 ++++++------- .../src/main/resources/common_tokens/sna | 13057 ++-- .../src/main/resources/common_tokens/snd | 29833 ++++----- .../src/main/resources/common_tokens/som | 18438 +++--- .../src/main/resources/common_tokens/spa | 59330 +++++++++--------- .../src/main/resources/common_tokens/sqi | 59562 +++++++++--------- .../src/main/resources/common_tokens/srp | 59966 +++++++++--------- .../src/main/resources/common_tokens/stq | 7243 ++- .../src/main/resources/common_tokens/sun | 35409 ++++++----- .../src/main/resources/common_tokens/swe | 51742 ++++++++-------- .../src/main/resources/common_tokens/swh | 50058 +++++++++------- .../src/main/resources/common_tokens/szl | 13274 ++-- .../src/main/resources/common_tokens/szy | 10252 ++-- .../src/main/resources/common_tokens/tam | 30387 +++++----- .../src/main/resources/common_tokens/tat | 54684 ++++++++--------- .../src/main/resources/common_tokens/tay | 2576 +- .../src/main/resources/common_tokens/tel | 32278 +++++----- .../src/main/resources/common_tokens/tet | 42675 ++++++------- .../src/main/resources/common_tokens/tgk | 55669 +++++++++-------- .../src/main/resources/common_tokens/tgl | 59024 +++++++++--------- .../src/main/resources/common_tokens/tha | 47313 ++++++++------- .../src/main/resources/common_tokens/tir | 59808 +++++++++--------- .../src/main/resources/common_tokens/trv | 6993 ++- .../src/main/resources/common_tokens/tsn | 24409 ++++---- .../src/main/resources/common_tokens/tso | 22879 +++---- .../src/main/resources/common_tokens/tuk | 23762 ++++---- .../src/main/resources/common_tokens/tum | 10273 ++-- .../src/main/resources/common_tokens/tur | 59726 +++++++++--------- .../src/main/resources/common_tokens/tyv | 12173 ++-- .../src/main/resources/common_tokens/udm | 58990 +++++++++--------- .../src/main/resources/common_tokens/uig | 38074 ++++++------ .../src/main/resources/common_tokens/ukr | 59798 +++++++++--------- .../src/main/resources/common_tokens/urd | 56988 +++++++++--------- .../src/main/resources/common_tokens/uzb | 59316 +++++++++--------- .../src/main/resources/common_tokens/vep | 15464 ++--- .../src/main/resources/common_tokens/vie | 36981 ++++++------ .../src/main/resources/common_tokens/vls | 17456 +++--- .../src/main/resources/common_tokens/vol | 7883 +-- .../src/main/resources/common_tokens/vro | 6011 +- .../src/main/resources/common_tokens/war | 44127 +++++++------- .../src/main/resources/common_tokens/wln | 14345 +++-- .../src/main/resources/common_tokens/xho | 58710 +++++++++--------- .../src/main/resources/common_tokens/xmf | 21162 ++++--- .../src/main/resources/common_tokens/ydd | 26151 ++++---- .../src/main/resources/common_tokens/yor | 11592 ++-- .../src/main/resources/common_tokens/yue | 23964 +++++--- .../src/main/resources/common_tokens/zho | 60008 +++++++++---------- .../src/main/resources/common_tokens/zul | 10937 ++-- .../core/tokens/tools/CommonTokenGenerator.java | 7 +- .../tika/langdetect/charsoup/CharSoupModel.java | 59 +- .../charsoup/GenerativeLanguageModel.java | 953 +- .../langdetect/charsoup/GlmScriptCategory.java | 174 + .../charsoup/SaltedNgramFeatureExtractor.java | 502 + .../charsoup/ScriptAwareFeatureExtractor.java | 79 +- .../tika/langdetect/charsoup/ScriptCategory.java | 44 +- .../charsoup/ShortTextFeatureExtractor.java | 70 +- .../langdetect/charsoup/langdetect-20260320.bin | Bin 0 -> 6687361 bytes .../charsoup/langdetect-generative-v1-20260310.bin | Bin 6679371 -> 0 bytes .../charsoup/langdetect-generative-v4-20260320.bin | Bin 0 -> 7493241 bytes .../charsoup/langdetect-short-v1-20260310.bin | Bin 3999308 -> 0 bytes .../langdetect/charsoup/langdetect-v7-20260306.bin | Bin 3328628 -> 0 bytes .../charsoup/CharSoupDetectorConfig.java | 60 +- .../charsoup/CharSoupLanguageDetector.java | 523 +- .../charsoup/CharSoupDetectorConfigTest.java | 44 +- .../charsoup/CharSoupModelRoutingTest.java | 153 +- .../langdetect/charsoup/LangIdRegressionTest.java | 26 +- .../charsoup/ScriptAwareFeatureExtractorTest.java | 82 +- .../charsoup/tools/CompareDetectors.java | 103 +- .../langdetect/charsoup/tools/CorpusAliases.java | 79 + .../tika/langdetect/charsoup/tools/FloresNorm.java | 109 + .../charsoup/tools/GlmAdjudicateDiagnostic.java | 149 + .../charsoup/tools/GlmNoiseSensitivityReport.java | 469 + .../langdetect/charsoup/tools/GlmRerankerEval.java | 311 + .../charsoup/tools/MarginDiagnostic.java | 94 + .../langdetect/charsoup/tools/ModelQuantizer.java | 13 +- .../langdetect/charsoup/tools/Phase2Trainer.java | 110 +- .../langdetect/charsoup/tools/PrepareCorpus.java | 127 +- .../langdetect/charsoup/tools/PrintDiscLabels.java | 44 +- .../langdetect/charsoup/tools/RecalibrateGlm.java | 197 + .../charsoup/tools/ResearchFeatureExtractor.java | 67 +- .../tools/TrainGenerativeLanguageModel.java | 75 +- .../charsoup/tools/TrainLanguageModel.java | 44 +- .../langdetect/charsoup/tools/TrainShortModel.java | 179 - tika-parent/pom.xml | 44 +- .../tika/parser/image/AbstractImageParser.java | 23 +- .../parser/microsoft/AbstractPOIFSExtractor.java | 2 +- .../tika/parser/microsoft/OutlookExtractor.java | 154 +- .../msg/RTFEncapsulatedHTMLExtractor.java | 453 + .../parser/microsoft/ooxml/InlineTagManager.java | 196 + .../microsoft/ooxml/ParagraphProperties.java | 9 + .../ooxml/XSSFExcelExtractorDecorator.java | 23 +- .../tika/parser/microsoft/OutlookParserTest.java | 43 + .../msg/RTFEncapsulatedHTMLExtractorTest.java | 217 + .../resources/test-documents/testWORD_2006ml.docx | Bin 165566 -> 151733 bytes .../apache/tika/parser/ocr/ImagePreprocessor.java | 4 +- .../apache/tika/parser/ocr/TesseractOCRParser.java | 49 +- .../tika/parser/ocr/TesseractOCRParserTest.java | 4 +- .../java/org/apache/tika/parser/pkg/ZipParser.java | 4 + .../org/apache/tika/async/cli/TikaAsyncCLI.java | 4 +- tika-pipes/tika-pipes-config-store-ignite/pom.xml | 2 +- .../tika/pipes/core/PerClientServerManager.java | 14 +- .../org/apache/tika/pipes/core/PipesClient.java | 4 +- .../tika/pipes/core/SharedServerManager.java | 4 +- .../tika/pipes/core/server/ConnectionHandler.java | 11 +- .../apache/tika/pipes/core/server/EmitHandler.java | 10 +- .../tika/pipes/core/server/FetchHandler.java | 4 +- .../tika/pipes/core/server/ParseHandler.java | 22 +- .../apache/tika/pipes/core/server/PipesServer.java | 18 +- .../apache/tika/pipes/core/server/PipesWorker.java | 4 +- .../resources/pipes-fork-server-default-log4j2.xml | 2 +- .../tika-pipes-google-drive/pom.xml | 2 +- .../tika/pipes/fetcher/http/HttpFetcherTest.java | 6 +- .../tika-pipes-microsoft-graph/pom.xml | 3 +- tika-server/tika-server-core/pom.xml | 6 +- .../org/apache/tika/server/core/CXFTestBase.java | 65 +- .../apache/tika/server/core/StackTraceTest.java | 8 +- .../server/core/TikaServerIntegrationTest.java | 22 + 296 files changed, 3612239 insertions(+), 3370505 deletions(-)
