You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
676 lines
29 KiB
676 lines
29 KiB
From f247f08e370626bbb427acd8f4a400fd875350a3 Mon Sep 17 00:00:00 2001
|
|
Message-Id: <f247f08e370626bbb427acd8f4a400fd875350a3.1531842113.git.erack@redhat.com>
|
|
From: Eike Rathke <erack@redhat.com>
|
|
Date: Tue, 17 Apr 2018 20:13:52 +0200
|
|
Subject: [PATCH] Upgrade to ICU 61.1
|
|
MIME-Version: 1.0
|
|
Content-Type: multipart/mixed; boundary="------------erAck-patch-parts"
|
|
|
|
This is a multi-part message in MIME format.
|
|
--------------erAck-patch-parts
|
|
Content-Type: text/plain; charset=UTF-8; format=fixed
|
|
Content-Transfer-Encoding: 8bit
|
|
|
|
|
|
Change-Id: I89c1c3d13d85decc72576744de2a16d20471d29d
|
|
Reviewed-on: https://gerrit.libreoffice.org/53064
|
|
Tested-by: Jenkins <ci@libreoffice.org>
|
|
Reviewed-by: Eike Rathke <erack@redhat.com>
|
|
---
|
|
configure.ac | 4 +-
|
|
download.lst | 8 +-
|
|
external/icu/UnpackedTarball_icu.mk | 3 +-
|
|
...patch.1 => icu4c-61-werror-shadow.patch.1} | 30 ++-
|
|
external/icu/icu4c-khmerbreakengine.patch.1 | 246 +++++++++---------
|
|
external/icu/icu4c-ubsan.patch.1 | 91 -------
|
|
6 files changed, 147 insertions(+), 235 deletions(-)
|
|
rename external/icu/{icu4c-59-werror-shadow.patch.1 => icu4c-61-werror-shadow.patch.1} (50%)
|
|
delete mode 100644 external/icu/icu4c-ubsan.patch.1
|
|
|
|
|
|
--------------erAck-patch-parts
|
|
Content-Type: text/x-patch; name="0001-Upgrade-to-ICU-61.1.patch"
|
|
Content-Transfer-Encoding: 8bit
|
|
Content-Disposition: attachment; filename="0001-Upgrade-to-ICU-61.1.patch"
|
|
|
|
diff --git a/configure.ac b/configure.ac
|
|
index bdc374699cb9..458f9f3c2735 100644
|
|
--- a/configure.ac
|
|
+++ b/configure.ac
|
|
@@ -9001,8 +9001,8 @@ SYSTEM_GENBRK=
|
|
SYSTEM_GENCCODE=
|
|
SYSTEM_GENCMN=
|
|
|
|
-ICU_MAJOR=60
|
|
-ICU_MINOR=2
|
|
+ICU_MAJOR=61
|
|
+ICU_MINOR=1
|
|
ICU_RECLASSIFIED_PREPEND_SET_EMPTY="TRUE"
|
|
ICU_RECLASSIFIED_CONDITIONAL_JAPANESE_STARTER="TRUE"
|
|
ICU_RECLASSIFIED_HEBREW_LETTER="TRUE"
|
|
diff --git a/download.lst b/download.lst
|
|
index 0a1fa65396d1..76f067bff4d7 100644
|
|
--- a/download.lst
|
|
+++ b/download.lst
|
|
@@ -102,10 +102,10 @@ export HUNSPELL_SHA256SUM := 3cd9ceb062fe5814f668e4f22b2fa6e3ba0b339b921739541ce
|
|
export HUNSPELL_TARBALL := hunspell-1.6.2.tar.gz
|
|
export HYPHEN_SHA256SUM := 304636d4eccd81a14b6914d07b84c79ebb815288c76fe027b9ebff6ff24d5705
|
|
export HYPHEN_TARBALL := 5ade6ae2a99bc1e9e57031ca88d36dad-hyphen-2.8.8.tar.gz
|
|
-export ICU_SHA256SUM := f073ea8f35b926d70bb33e6577508aa642a8b316a803f11be20af384811db418
|
|
-export ICU_TARBALL := icu4c-60_2-src.tgz
|
|
-export ICU_DATA_SHA256SUM := 68f42ad0c9e0a5a5af8eba0577ba100833912288bad6e4d1f42ff480bbcfd4a9
|
|
-export ICU_DATA_TARBALL := icu4c-60_2-data.zip
|
|
+export ICU_SHA256SUM := d007f89ae8a2543a53525c74359b65b36412fa84b3349f1400be6dcf409fafef
|
|
+export ICU_TARBALL := icu4c-61_1-src.tgz
|
|
+export ICU_DATA_SHA256SUM := d149ed0985b5a6e16a9d8ed66f105dd58fd334c276779f74241cfa656ed2830a
|
|
+export ICU_DATA_TARBALL := icu4c-61_1-data.zip
|
|
export JFREEREPORT_FLOW_ENGINE_SHA256SUM := 233f66e8d25c5dd971716d4200203a612a407649686ef3b52075d04b4c9df0dd
|
|
export JFREEREPORT_FLOW_ENGINE_TARBALL := ba2930200c9f019c2d93a8c88c651a0f-flow-engine-0.9.4.zip
|
|
export JFREEREPORT_FLUTE_SHA256SUM := 1b5b24f7bc543c0362b667692f78db8bab4ed6dafc6172f104d0bd3757d8a133
|
|
diff --git a/external/icu/UnpackedTarball_icu.mk b/external/icu/UnpackedTarball_icu.mk
|
|
index a4d0b16ecb36..b81cdaab6242 100644
|
|
--- a/external/icu/UnpackedTarball_icu.mk
|
|
+++ b/external/icu/UnpackedTarball_icu.mk
|
|
@@ -27,15 +27,14 @@ $(eval $(call gb_UnpackedTarball_add_patches,icu,\
|
|
external/icu/icu4c-solarisgcc.patch.1 \
|
|
external/icu/icu4c-mkdir.patch.1 \
|
|
external/icu/icu4c-$(if $(filter ANDROID,$(OS)),android,rpath).patch.1 \
|
|
- external/icu/icu4c-ubsan.patch.1 \
|
|
external/icu/icu4c-icu11100.patch.1 \
|
|
external/icu/icu4c-scriptrun.patch.1 \
|
|
external/icu/icu4c-rtti.patch.1 \
|
|
external/icu/icu4c-clang-cl.patch.1 \
|
|
$(if $(filter-out ANDROID,$(OS)),external/icu/icu4c-icudata-stdlibs.patch.1) \
|
|
external/icu/icu4c-khmerbreakengine.patch.1 \
|
|
- external/icu/icu4c-59-werror-shadow.patch.1 \
|
|
external/icu/ofz4860.patch.2 \
|
|
+ external/icu/icu4c-61-werror-shadow.patch.1 \
|
|
))
|
|
|
|
$(eval $(call gb_UnpackedTarball_add_file,icu,source/data/brkitr/khmerdict.dict,external/icu/khmerdict.dict))
|
|
diff --git a/external/icu/icu4c-59-werror-shadow.patch.1 b/external/icu/icu4c-61-werror-shadow.patch.1
|
|
similarity index 50%
|
|
rename from external/icu/icu4c-59-werror-shadow.patch.1
|
|
rename to external/icu/icu4c-61-werror-shadow.patch.1
|
|
index fb88244aff13..b00f76317eff 100644
|
|
--- a/external/icu/icu4c-59-werror-shadow.patch.1
|
|
+++ b/external/icu/icu4c-61-werror-shadow.patch.1
|
|
@@ -1,33 +1,35 @@
|
|
+# https://ssl.icu-project.org/trac/ticket/13709
|
|
+# Werror=shadow fails for unistr.h
|
|
diff -ur icu.org/source/common/unicode/unistr.h icu/source/common/unicode/unistr.h
|
|
---- icu.org/source/common/unicode/unistr.h 2017-03-29 06:44:37.000000000 +0200
|
|
-+++ icu/source/common/unicode/unistr.h 2017-04-24 11:59:51.782076511 +0200
|
|
-@@ -3080,11 +3080,11 @@
|
|
+--- icu.org/source/common/unicode/unistr.h 2018-03-26 15:38:29.000000000 +0200
|
|
++++ icu/source/common/unicode/unistr.h 2018-04-18 10:44:16.321188314 +0200
|
|
+@@ -3053,11 +3053,11 @@
|
|
* uint16_t * constructor.
|
|
* Delegates to UnicodeString(const char16_t *, int32_t).
|
|
* @param text UTF-16 string
|
|
- * @param length string length
|
|
-+ * @param length_ string length
|
|
- * @draft ICU 59
|
|
++ * @param textLength string length
|
|
+ * @stable ICU 59
|
|
*/
|
|
- UnicodeString(const uint16_t *text, int32_t length) :
|
|
- UnicodeString(ConstChar16Ptr(text), length) {}
|
|
-+ UnicodeString(const uint16_t *text, int32_t length_) :
|
|
-+ UnicodeString(ConstChar16Ptr(text), length_) {}
|
|
++ UnicodeString(const uint16_t *text, int32_t textLength) :
|
|
++ UnicodeString(ConstChar16Ptr(text), textLength) {}
|
|
#endif
|
|
|
|
- /*
|
|
-@@ -3097,11 +3097,11 @@
|
|
+ #if U_SIZEOF_WCHAR_T==2 || defined(U_IN_DOXYGEN)
|
|
+@@ -3066,11 +3066,11 @@
|
|
* (Only defined if U_SIZEOF_WCHAR_T==2.)
|
|
* Delegates to UnicodeString(const char16_t *, int32_t).
|
|
* @param text NUL-terminated UTF-16 string
|
|
- * @param length string length
|
|
-+ * @param length_ string length
|
|
- * @draft ICU 59
|
|
++ * @param textLength string length
|
|
+ * @stable ICU 59
|
|
*/
|
|
- UnicodeString(const wchar_t *text, int32_t length) :
|
|
- UnicodeString(ConstChar16Ptr(text), length) {}
|
|
-+ UnicodeString(const wchar_t *text, int32_t length_) :
|
|
-+ UnicodeString(ConstChar16Ptr(text), length_) {}
|
|
++ UnicodeString(const wchar_t *text, int32_t textLength) :
|
|
++ UnicodeString(ConstChar16Ptr(text), textLength) {}
|
|
#endif
|
|
|
|
- /*
|
|
+ /**
|
|
diff --git a/external/icu/icu4c-khmerbreakengine.patch.1 b/external/icu/icu4c-khmerbreakengine.patch.1
|
|
index 6b45b3743611..9f134dd961b1 100644
|
|
--- a/external/icu/icu4c-khmerbreakengine.patch.1
|
|
+++ b/external/icu/icu4c-khmerbreakengine.patch.1
|
|
@@ -1,16 +1,18 @@
|
|
diff -ur icu.org/source/common/dictbe.cpp icu/source/common/dictbe.cpp
|
|
---- icu.org/source/common/dictbe.cpp 2017-01-20 01:20:31.000000000 +0100
|
|
-+++ icu/source/common/dictbe.cpp 2017-04-21 23:14:23.845894374 +0200
|
|
-@@ -29,8 +29,17 @@
|
|
+--- icu.org/source/common/dictbe.cpp 2018-03-26 15:38:30.000000000 +0200
|
|
++++ icu/source/common/dictbe.cpp 2018-04-17 17:55:38.620944919 +0200
|
|
+@@ -29,7 +29,19 @@
|
|
******************************************************************
|
|
*/
|
|
|
|
--DictionaryBreakEngine::DictionaryBreakEngine(uint32_t breakTypes) {
|
|
-+DictionaryBreakEngine::DictionaryBreakEngine(uint32_t breakTypes) :
|
|
-+ clusterLimit(3)
|
|
-+{
|
|
+-DictionaryBreakEngine::DictionaryBreakEngine() {
|
|
++DictionaryBreakEngine::DictionaryBreakEngine()
|
|
++ : fTypes(0), clusterLimit(0) {
|
|
++}
|
|
++
|
|
++DictionaryBreakEngine::DictionaryBreakEngine(uint32_t breakTypes)
|
|
++ : fTypes(breakTypes), clusterLimit(3) {
|
|
+ UErrorCode status = U_ZERO_ERROR;
|
|
- fTypes = breakTypes;
|
|
+ fViramaSet.applyPattern(UNICODE_STRING_SIMPLE("[[:ccc=VR:]]"), status);
|
|
+
|
|
+ // note Skip Sets contain fIgnoreSet characters too.
|
|
@@ -20,16 +22,7 @@ diff -ur icu.org/source/common/dictbe.cpp icu/source/common/dictbe.cpp
|
|
}
|
|
|
|
DictionaryBreakEngine::~DictionaryBreakEngine() {
|
|
-@@ -92,7 +101,7 @@
|
|
- result = divideUpDictionaryRange(text, rangeStart, rangeEnd, foundBreaks);
|
|
- utext_setNativeIndex(text, current);
|
|
- }
|
|
--
|
|
-+
|
|
- return result;
|
|
- }
|
|
-
|
|
-@@ -103,6 +112,169 @@
|
|
+@@ -76,6 +88,169 @@
|
|
fSet.compact();
|
|
}
|
|
|
|
@@ -199,7 +192,7 @@ diff -ur icu.org/source/common/dictbe.cpp icu/source/common/dictbe.cpp
|
|
/*
|
|
******************************************************************
|
|
* PossibleWord
|
|
-@@ -130,35 +302,35 @@
|
|
+@@ -103,35 +278,35 @@
|
|
public:
|
|
PossibleWord() : count(0), prefix(0), offset(-1), mark(0), current(0) {};
|
|
~PossibleWord() {};
|
|
@@ -244,7 +237,7 @@ diff -ur icu.org/source/common/dictbe.cpp icu/source/common/dictbe.cpp
|
|
// Dictionary leaves text after longest prefix, not longest word. Back up.
|
|
if (count <= 0) {
|
|
utext_setNativeIndex(text, start);
|
|
-@@ -830,51 +1002,28 @@
|
|
+@@ -803,51 +978,28 @@
|
|
* KhmerBreakEngine
|
|
*/
|
|
|
|
@@ -265,7 +258,8 @@ diff -ur icu.org/source/common/dictbe.cpp icu/source/common/dictbe.cpp
|
|
-static const int32_t KHMER_MIN_WORD_SPAN = KHMER_MIN_WORD * 2;
|
|
-
|
|
KhmerBreakEngine::KhmerBreakEngine(DictionaryMatcher *adoptDictionary, UErrorCode &status)
|
|
- : DictionaryBreakEngine((1 << UBRK_WORD) | (1 << UBRK_LINE)),
|
|
+- : DictionaryBreakEngine(),
|
|
++ : DictionaryBreakEngine((1 << UBRK_WORD) | (1 << UBRK_LINE)),
|
|
fDictionary(adoptDictionary)
|
|
{
|
|
- fKhmerWordSet.applyPattern(UNICODE_STRING_SIMPLE("[[:Khmr:]&[:LineBreak=SA:]]"), status);
|
|
@@ -301,13 +295,13 @@ diff -ur icu.org/source/common/dictbe.cpp icu/source/common/dictbe.cpp
|
|
- fEndWordSet.compact();
|
|
- fBeginWordSet.compact();
|
|
-// fSuffixSet.compact();
|
|
-+ fIgnoreSet.compact();
|
|
-+ fBaseSet.compact();
|
|
-+ fPuncSet.compact();
|
|
++ fIgnoreSet.compact();
|
|
++ fBaseSet.compact();
|
|
++ fPuncSet.compact();
|
|
}
|
|
|
|
KhmerBreakEngine::~KhmerBreakEngine() {
|
|
-@@ -886,180 +1035,204 @@
|
|
+@@ -859,180 +1011,204 @@
|
|
int32_t rangeStart,
|
|
int32_t rangeEnd,
|
|
UVector32 &foundBreaks ) const {
|
|
@@ -350,17 +344,6 @@ diff -ur icu.org/source/common/dictbe.cpp icu/source/common/dictbe.cpp
|
|
+ foundBreaks.push(rangeEnd, status);
|
|
+ return foundBreaks.size() - wordsFound;
|
|
+ }
|
|
-+
|
|
-+ scanStart = rangeStart;
|
|
-+ scanWJ(text, scanStart, rangeEnd, before, after);
|
|
-+ if (startZwsp || initAfter >= before) {
|
|
-+ after = initAfter;
|
|
-+ before = 0;
|
|
-+ }
|
|
-+ if (!endZwsp && after > finalBefore && after < rangeEnd)
|
|
-+ endZwsp = true;
|
|
-+ if (endZwsp && before > finalBefore)
|
|
-+ before = finalBefore;
|
|
|
|
- while (U_SUCCESS(status) && (current = (int32_t)utext_getNativeIndex(text)) < rangeEnd) {
|
|
- cuWordLength = 0;
|
|
@@ -375,7 +358,17 @@ diff -ur icu.org/source/common/dictbe.cpp icu/source/common/dictbe.cpp
|
|
- cpWordLength = words[wordsFound % KHMER_LOOKAHEAD].markedCPLength();
|
|
- wordsFound += 1;
|
|
- }
|
|
--
|
|
++ scanStart = rangeStart;
|
|
++ scanWJ(text, scanStart, rangeEnd, before, after);
|
|
++ if (startZwsp || initAfter >= before) {
|
|
++ after = initAfter;
|
|
++ before = 0;
|
|
++ }
|
|
++ if (!endZwsp && after > finalBefore && after < rangeEnd)
|
|
++ endZwsp = true;
|
|
++ if (endZwsp && before > finalBefore)
|
|
++ before = finalBefore;
|
|
+
|
|
- // If there was more than one, see which one can take us forward the most words
|
|
- else if (candidates > 1) {
|
|
- // If we're already at the end of the range, we're done
|
|
@@ -390,22 +383,6 @@ diff -ur icu.org/source/common/dictbe.cpp icu/source/common/dictbe.cpp
|
|
- words[wordsFound % KHMER_LOOKAHEAD].markCurrent();
|
|
- wordsMatched = 2;
|
|
- }
|
|
--
|
|
-- // If we're already at the end of the range, we're done
|
|
-- if ((int32_t)utext_getNativeIndex(text) >= rangeEnd) {
|
|
-- goto foundBest;
|
|
-- }
|
|
--
|
|
-- // See if any of the possible second words is followed by a third word
|
|
-- do {
|
|
-- // If we find a third word, stop right away
|
|
-- if (words[(wordsFound + 2) % KHMER_LOOKAHEAD].candidates(text, fDictionary, rangeEnd)) {
|
|
-- words[wordsFound % KHMER_LOOKAHEAD].markCurrent();
|
|
-- goto foundBest;
|
|
-- }
|
|
-- }
|
|
-- while (words[(wordsFound + 1) % KHMER_LOOKAHEAD].backUp(text));
|
|
-- }
|
|
+ utext_setNativeIndex(text, rangeStart);
|
|
+ int32_t numCodePts = rangeEnd - rangeStart;
|
|
+ // bestSnlp[i] is the snlp of the best segmentation of the first i
|
|
@@ -415,7 +392,11 @@ diff -ur icu.org/source/common/dictbe.cpp icu/source/common/dictbe.cpp
|
|
+ for(int32_t i = 1; i <= numCodePts; i++) {
|
|
+ bestSnlp.addElement(kuint32max, status);
|
|
+ }
|
|
-+
|
|
+
|
|
+- // If we're already at the end of the range, we're done
|
|
+- if ((int32_t)utext_getNativeIndex(text) >= rangeEnd) {
|
|
+- goto foundBest;
|
|
+- }
|
|
+ // prev[i] is the index of the last code point in the previous word in
|
|
+ // the best segmentation of the first i characters. Note negative implies
|
|
+ // that the code point is part of an unknown word.
|
|
@@ -423,7 +404,17 @@ diff -ur icu.org/source/common/dictbe.cpp icu/source/common/dictbe.cpp
|
|
+ for(int32_t i = 0; i <= numCodePts; i++) {
|
|
+ prev.addElement(kuint32max, status);
|
|
+ }
|
|
-+
|
|
+
|
|
+- // See if any of the possible second words is followed by a third word
|
|
+- do {
|
|
+- // If we find a third word, stop right away
|
|
+- if (words[(wordsFound + 2) % KHMER_LOOKAHEAD].candidates(text, fDictionary, rangeEnd)) {
|
|
+- words[wordsFound % KHMER_LOOKAHEAD].markCurrent();
|
|
+- goto foundBest;
|
|
+- }
|
|
+- }
|
|
+- while (words[(wordsFound + 1) % KHMER_LOOKAHEAD].backUp(text));
|
|
+- }
|
|
+ const int32_t maxWordSize = 20;
|
|
+ UVector32 values(maxWordSize, status);
|
|
+ values.setSize(maxWordSize);
|
|
@@ -528,27 +519,17 @@ diff -ur icu.org/source/common/dictbe.cpp icu/source/common/dictbe.cpp
|
|
- if (cuWordLength <= 0) {
|
|
- wordsFound += 1;
|
|
- }
|
|
--
|
|
-- // Update the length with the passed-over characters
|
|
-- cuWordLength += chars;
|
|
-- }
|
|
-- else {
|
|
-- // Back up to where we were for next iteration
|
|
-- utext_setNativeIndex(text, current+cuWordLength);
|
|
+ } while (fMarkSet.contains(c) || fIgnoreSet.contains(c));
|
|
+ values.setElementAt(BADSNLP, count);
|
|
+ lengths.setElementAt(utext_getNativeIndex(text) - currix, count++);
|
|
+ } else {
|
|
+ values.setElementAt(BADSNLP, count);
|
|
+ lengths.setElementAt(1, count++);
|
|
- }
|
|
- }
|
|
++ }
|
|
++ }
|
|
|
|
-- // Never stop before a combining mark.
|
|
-- int32_t currPos;
|
|
-- while ((currPos = (int32_t)utext_getNativeIndex(text)) < rangeEnd && fMarkSet.contains(utext_current32(text))) {
|
|
-- utext_next32(text);
|
|
-- cuWordLength += (int32_t)utext_getNativeIndex(text) - currPos;
|
|
+- // Update the length with the passed-over characters
|
|
+- cuWordLength += chars;
|
|
+ for (int32_t j = 0; j < count; j++) {
|
|
+ uint32_t v = values.elementAti(j);
|
|
+ int32_t newSnlp = bestSnlp.elementAti(i) + v;
|
|
@@ -559,7 +540,10 @@ diff -ur icu.org/source/common/dictbe.cpp icu/source/common/dictbe.cpp
|
|
+ ++ln;
|
|
+ utext_next32(text);
|
|
+ c = utext_current32(text);
|
|
-+ }
|
|
+ }
|
|
+- else {
|
|
+- // Back up to where we were for next iteration
|
|
+- utext_setNativeIndex(text, current+cuWordLength);
|
|
+ int32_t ln_j_i = ln + i; // yes really i!
|
|
+ if (newSnlp < bestSnlp.elementAti(ln_j_i)) {
|
|
+ if (v == BADSNLP) {
|
|
@@ -572,9 +556,38 @@ diff -ur icu.org/source/common/dictbe.cpp icu/source/common/dictbe.cpp
|
|
+ else
|
|
+ prev.setElementAt(i, ln_j_i);
|
|
+ bestSnlp.setElementAt(newSnlp, ln_j_i);
|
|
-+ }
|
|
+ }
|
|
}
|
|
-
|
|
+- // Never stop before a combining mark.
|
|
+- int32_t currPos;
|
|
+- while ((currPos = (int32_t)utext_getNativeIndex(text)) < rangeEnd && fMarkSet.contains(utext_current32(text))) {
|
|
+- utext_next32(text);
|
|
+- cuWordLength += (int32_t)utext_getNativeIndex(text) - currPos;
|
|
++ }
|
|
++ // Start pushing the optimal offset index into t_boundary (t for tentative).
|
|
++ // prev[numCodePts] is guaranteed to be meaningful.
|
|
++ // We'll first push in the reverse order, i.e.,
|
|
++ // t_boundary[0] = numCodePts, and afterwards do a swap.
|
|
++ UVector32 t_boundary(numCodePts+1, status);
|
|
++
|
|
++ int32_t numBreaks = 0;
|
|
++ // No segmentation found, set boundary to end of range
|
|
++ while (numCodePts >= 0 && (uint32_t)bestSnlp.elementAti(numCodePts) == kuint32max) {
|
|
++ --numCodePts;
|
|
++ }
|
|
++ if (numCodePts < 0) {
|
|
++ t_boundary.addElement(numCodePts, status);
|
|
++ numBreaks++;
|
|
++ } else {
|
|
++ for (int32_t i = numCodePts; (uint32_t)i != kuint32max; i = prev.elementAti(i)) {
|
|
++ if (i < 0) i = -i;
|
|
++ t_boundary.addElement(i, status);
|
|
++ numBreaks++;
|
|
+ }
|
|
++ U_ASSERT(prev.elementAti(t_boundary.elementAti(numBreaks - 1)) == 0);
|
|
++ }
|
|
+
|
|
- // Look ahead for possible suffixes if a dictionary word does not follow.
|
|
- // We do this in code rather than using a rule so that the heuristic
|
|
- // resynch continues to function. For example, one of the suffix characters
|
|
@@ -616,30 +629,6 @@ diff -ur icu.org/source/common/dictbe.cpp icu/source/common/dictbe.cpp
|
|
- // Did we find a word on this iteration? If so, push it on the break stack
|
|
- if (cuWordLength > 0) {
|
|
- foundBreaks.push((current+cuWordLength), status);
|
|
-+ }
|
|
-+ // Start pushing the optimal offset index into t_boundary (t for tentative).
|
|
-+ // prev[numCodePts] is guaranteed to be meaningful.
|
|
-+ // We'll first push in the reverse order, i.e.,
|
|
-+ // t_boundary[0] = numCodePts, and afterwards do a swap.
|
|
-+ UVector32 t_boundary(numCodePts+1, status);
|
|
-+
|
|
-+ int32_t numBreaks = 0;
|
|
-+ // No segmentation found, set boundary to end of range
|
|
-+ while (numCodePts >= 0 && (uint32_t)bestSnlp.elementAti(numCodePts) == kuint32max) {
|
|
-+ --numCodePts;
|
|
-+ }
|
|
-+ if (numCodePts < 0) {
|
|
-+ t_boundary.addElement(numCodePts, status);
|
|
-+ numBreaks++;
|
|
-+ } else {
|
|
-+ for (int32_t i = numCodePts; (uint32_t)i != kuint32max; i = prev.elementAti(i)) {
|
|
-+ if (i < 0) i = -i;
|
|
-+ t_boundary.addElement(i, status);
|
|
-+ numBreaks++;
|
|
-+ }
|
|
-+ U_ASSERT(prev.elementAti(t_boundary.elementAti(numBreaks - 1)) == 0);
|
|
-+ }
|
|
-+
|
|
+ // Now that we're done, convert positions in t_boundary[] (indices in
|
|
+ // the normalized input string) back to indices in the original input UText
|
|
+ // while reversing t_boundary and pushing values to foundBreaks.
|
|
@@ -669,38 +658,35 @@ diff -ur icu.org/source/common/dictbe.cpp icu/source/common/dictbe.cpp
|
|
|
|
#if !UCONFIG_NO_NORMALIZATION
|
|
diff -ur icu.org/source/common/dictbe.h icu/source/common/dictbe.h
|
|
---- icu.org/source/common/dictbe.h 2017-01-20 01:20:31.000000000 +0100
|
|
-+++ icu/source/common/dictbe.h 2017-04-21 23:14:23.845894374 +0200
|
|
-@@ -34,6 +34,15 @@
|
|
+--- icu.org/source/common/dictbe.h 2018-03-26 15:38:30.000000000 +0200
|
|
++++ icu/source/common/dictbe.h 2018-04-17 14:55:33.307639865 +0200
|
|
+@@ -34,7 +34,8 @@
|
|
+ * threads without synchronization.</p>
|
|
*/
|
|
class DictionaryBreakEngine : public LanguageBreakEngine {
|
|
- private:
|
|
-+
|
|
-+ /**
|
|
-+ * <p>Default constructor.</p>
|
|
-+ *
|
|
-+ */
|
|
-+ DictionaryBreakEngine();
|
|
-+
|
|
+- private:
|
|
+ protected:
|
|
+
|
|
/**
|
|
* The set of characters handled by this engine
|
|
* @internal
|
|
-@@ -48,11 +57,63 @@
|
|
+@@ -42,14 +43,84 @@
|
|
|
|
- uint32_t fTypes;
|
|
+ UnicodeSet fSet;
|
|
|
|
+ const int32_t WJ = 0x2060;
|
|
+ const int32_t ZWSP = 0x200B;
|
|
+
|
|
- /**
|
|
-- * <p>Default constructor.</p>
|
|
-- *
|
|
++ /**
|
|
++ * The break types it was constructed with
|
|
++ * @internal
|
|
++ */
|
|
++ uint32_t fTypes;
|
|
++
|
|
++ /**
|
|
+ * A Unicode set of all viramas
|
|
+ * @internal
|
|
- */
|
|
-- DictionaryBreakEngine();
|
|
++ */
|
|
+ UnicodeSet fViramaSet;
|
|
+
|
|
+ /**
|
|
@@ -751,10 +737,26 @@ diff -ur icu.org/source/common/dictbe.h icu/source/common/dictbe.h
|
|
+ bool scanAfterEnd(UText *text, int32_t rangeEnd, int32_t& end, bool &doBreak) const;
|
|
+ void scanBackClusters(UText *text, int32_t textStart, int32_t& start) const;
|
|
+ void scanFwdClusters(UText *text, int32_t textEnd, int32_t& end) const;
|
|
-
|
|
++
|
|
public:
|
|
|
|
-@@ -83,7 +144,7 @@
|
|
+ /**
|
|
+- * <p>Constructor </p>
|
|
++ * <p>Default constructor.</p>
|
|
++ *
|
|
+ */
|
|
+ DictionaryBreakEngine();
|
|
+
|
|
+ /**
|
|
++ * <p>Constructor with break types.</p>
|
|
++ */
|
|
++ explicit DictionaryBreakEngine(uint32_t breakTypes);
|
|
++
|
|
++ /**
|
|
+ * <p>Virtual destructor.</p>
|
|
+ */
|
|
+ virtual ~DictionaryBreakEngine();
|
|
+@@ -68,7 +139,7 @@
|
|
* <p>Find any breaks within a run in the supplied text.</p>
|
|
*
|
|
* @param text A UText representing the text. The iterator is left at
|
|
@@ -763,7 +765,7 @@ diff -ur icu.org/source/common/dictbe.h icu/source/common/dictbe.h
|
|
* that starts from the first character in the range.
|
|
* @param startPos The start of the run within the supplied text.
|
|
* @param endPos The end of the run within the supplied text.
|
|
-@@ -245,118 +306,120 @@
|
|
+@@ -218,118 +289,120 @@
|
|
|
|
};
|
|
|
|
@@ -997,8 +999,8 @@ diff -ur icu.org/source/common/dictbe.h icu/source/common/dictbe.h
|
|
|
|
/*******************************************************************
|
|
diff -ur icu.org/source/common/dictionarydata.cpp icu/source/common/dictionarydata.cpp
|
|
---- icu.org/source/common/dictionarydata.cpp 2017-01-20 01:20:31.000000000 +0100
|
|
-+++ icu/source/common/dictionarydata.cpp 2017-04-21 23:14:23.846894372 +0200
|
|
+--- icu.org/source/common/dictionarydata.cpp 2018-03-26 15:38:30.000000000 +0200
|
|
++++ icu/source/common/dictionarydata.cpp 2018-04-17 14:04:50.775567214 +0200
|
|
@@ -44,7 +44,7 @@
|
|
|
|
int32_t UCharsDictionaryMatcher::matches(UText *text, int32_t maxLength, int32_t limit,
|
|
@@ -1046,8 +1048,8 @@ diff -ur icu.org/source/common/dictionarydata.cpp icu/source/common/dictionaryda
|
|
if (values != NULL) {
|
|
values[wordCount] = bt.getValue();
|
|
diff -ur icu.org/source/common/dictionarydata.h icu/source/common/dictionarydata.h
|
|
---- icu.org/source/common/dictionarydata.h 2017-01-20 01:20:31.000000000 +0100
|
|
-+++ icu/source/common/dictionarydata.h 2017-04-21 23:14:23.846894372 +0200
|
|
+--- icu.org/source/common/dictionarydata.h 2018-03-26 15:38:30.000000000 +0200
|
|
++++ icu/source/common/dictionarydata.h 2018-04-17 14:04:50.775567214 +0200
|
|
@@ -21,6 +21,7 @@
|
|
#include "unicode/utext.h"
|
|
#include "unicode/udata.h"
|
|
@@ -1084,8 +1086,8 @@ diff -ur icu.org/source/common/dictionarydata.h icu/source/common/dictionarydata
|
|
private:
|
|
UChar32 transform(UChar32 c) const;
|
|
diff -ur icu.org/source/data/Makefile.in icu/source/data/Makefile.in
|
|
---- icu.org/source/data/Makefile.in 2017-04-21 23:13:03.248087545 +0200
|
|
-+++ icu/source/data/Makefile.in 2017-04-21 23:14:23.846894372 +0200
|
|
+--- icu.org/source/data/Makefile.in 2018-04-17 12:28:37.098707466 +0200
|
|
++++ icu/source/data/Makefile.in 2018-04-17 14:04:50.775567214 +0200
|
|
@@ -183,7 +183,7 @@
|
|
endif
|
|
endif
|
|
diff --git a/external/icu/icu4c-ubsan.patch.1 b/external/icu/icu4c-ubsan.patch.1
|
|
deleted file mode 100644
|
|
index 9f6aa3fbc9f8..000000000000
|
|
--- a/external/icu/icu4c-ubsan.patch.1
|
|
+++ /dev/null
|
|
@@ -1,91 +0,0 @@
|
|
-diff -ur icu.org/source/common/rbbidata.h icu/source/common/rbbidata.h
|
|
---- icu.org/source/common/rbbidata.h 2017-02-03 19:57:23.000000000 +0100
|
|
-+++ icu/source/common/rbbidata.h 2017-04-21 22:46:25.371651160 +0200
|
|
-@@ -115,7 +115,7 @@
|
|
- /* StatusTable of the set of matching */
|
|
- /* tags (rule status values) */
|
|
- int16_t fReserved;
|
|
-- uint16_t fNextState[2]; /* Next State, indexed by char category. */
|
|
-+ uint16_t fNextState[1]; /* Next State, indexed by char category. */
|
|
- /* This array does not have two elements */
|
|
- /* Array Size is actually fData->fHeader->fCatCount */
|
|
- /* CAUTION: see RBBITableBuilder::getTableSize() */
|
|
-@@ -128,7 +128,7 @@
|
|
- uint32_t fRowLen; /* Length of a state table row, in bytes. */
|
|
- uint32_t fFlags; /* Option Flags for this state table */
|
|
- uint32_t fReserved; /* reserved */
|
|
-- char fTableData[4]; /* First RBBIStateTableRow begins here. */
|
|
-+ char fTableData[1]; /* First RBBIStateTableRow begins here. */
|
|
- /* (making it char[] simplifies ugly address */
|
|
- /* arithmetic for indexing variable length rows.) */
|
|
- };
|
|
-diff -ur icu.org/source/common/rbbitblb.cpp icu/source/common/rbbitblb.cpp
|
|
---- icu.org/source/common/rbbitblb.cpp 2017-01-20 01:20:31.000000000 +0100
|
|
-+++ icu/source/common/rbbitblb.cpp 2017-04-21 22:46:25.373651159 +0200
|
|
-@@ -1095,15 +1095,15 @@
|
|
- return 0;
|
|
- }
|
|
-
|
|
-- size = sizeof(RBBIStateTable) - 4; // The header, with no rows to the table.
|
|
-+ size = offsetof(RBBIStateTable, fTableData); // The header, with no rows to the table.
|
|
-
|
|
- numRows = fDStates->size();
|
|
- numCols = fRB->fSetBuilder->getNumCharCategories();
|
|
-
|
|
-- // Note The declaration of RBBIStateTableRow is for a table of two columns.
|
|
-- // Therefore we subtract two from numCols when determining
|
|
-+ // Note The declaration of RBBIStateTableRow is for a table of one columns.
|
|
-+ // Therefore we subtract one from numCols when determining
|
|
- // how much storage to add to a row for the total columns.
|
|
-- rowSize = sizeof(RBBIStateTableRow) + sizeof(uint16_t)*(numCols-2);
|
|
-+ rowSize = sizeof(RBBIStateTableRow) + sizeof(uint16_t)*(numCols-1);
|
|
- size += numRows * rowSize;
|
|
- return size;
|
|
- }
|
|
-@@ -1133,7 +1133,7 @@
|
|
- }
|
|
-
|
|
- table->fRowLen = sizeof(RBBIStateTableRow) +
|
|
-- sizeof(uint16_t) * (fRB->fSetBuilder->getNumCharCategories() - 2);
|
|
-+ sizeof(uint16_t) * (fRB->fSetBuilder->getNumCharCategories() - 1);
|
|
- table->fNumStates = fDStates->size();
|
|
- table->fFlags = 0;
|
|
- if (fRB->fLookAheadHardBreak) {
|
|
-diff -ur icu.org/source/common/ubidiimp.h icu/source/common/ubidiimp.h
|
|
---- icu.org/source/common/ubidiimp.h 2017-02-03 19:57:23.000000000 +0100
|
|
-+++ icu/source/common/ubidiimp.h 2017-04-21 22:46:25.374651159 +0200
|
|
-@@ -198,8 +198,8 @@
|
|
- /* in a Run, logicalStart will get this bit set if the run level is odd */
|
|
- #define INDEX_ODD_BIT (1UL<<31)
|
|
-
|
|
--#define MAKE_INDEX_ODD_PAIR(index, level) ((index)|((int32_t)(level)<<31))
|
|
--#define ADD_ODD_BIT_FROM_LEVEL(x, level) ((x)|=((int32_t)(level)<<31))
|
|
-+#define MAKE_INDEX_ODD_PAIR(index, level) ((index)|((uint32_t)(level)<<31))
|
|
-+#define ADD_ODD_BIT_FROM_LEVEL(x, level) ((x)|=((uint32_t)(level)<<31))
|
|
- #define REMOVE_ODD_BIT(x) ((x)&=~INDEX_ODD_BIT)
|
|
-
|
|
- #define GET_INDEX(x) ((x)&~INDEX_ODD_BIT)
|
|
-diff -ur icu.org/source/common/ucmndata.cpp icu/source/common/ucmndata.cpp
|
|
---- icu.org/source/common/ucmndata.cpp 2017-03-08 16:34:47.000000000 +0100
|
|
-+++ icu/source/common/ucmndata.cpp 2017-04-21 22:46:25.376651159 +0200
|
|
-@@ -77,7 +77,7 @@
|
|
- typedef struct {
|
|
- uint32_t count;
|
|
- uint32_t reserved;
|
|
-- PointerTOCEntry entry[2]; /* Actual size is from count. */
|
|
-+ PointerTOCEntry entry[1]; /* Actual size is from count. */
|
|
- } PointerTOC;
|
|
-
|
|
-
|
|
-diff -ur icu.org/source/common/ucmndata.h icu/source/common/ucmndata.h
|
|
---- icu.org/source/common/ucmndata.h 2017-01-20 01:20:31.000000000 +0100
|
|
-+++ icu/source/common/ucmndata.h 2017-04-21 22:46:25.377651159 +0200
|
|
-@@ -52,7 +52,7 @@
|
|
-
|
|
- typedef struct {
|
|
- uint32_t count;
|
|
-- UDataOffsetTOCEntry entry[2]; /* Actual size of array is from count. */
|
|
-+ UDataOffsetTOCEntry entry[1]; /* Actual size of array is from count. */
|
|
- } UDataOffsetTOC;
|
|
-
|
|
- /**
|
|
|
|
--------------erAck-patch-parts--
|
|
|
|
|