blob: 52e9c53621dca2843f0ccab85e15aa110957da09 [file] [log] [blame]
Jungshik Shin87232d82017-05-13 21:10:13 -07001// © 2016 and later: Unicode, Inc. and others.
Jungshik Shin5feb9ad2016-10-21 12:52:48 -07002// License & terms of use: http://www.unicode.org/copyright.html
jshin@chromium.org6f31ac32014-03-26 22:15:14 +00003/*
4 ************************************************************************************
Jungshik Shin5feb9ad2016-10-21 12:52:48 -07005 * Copyright (C) 2006-2016, International Business Machines Corporation
jshin@chromium.org6f31ac32014-03-26 22:15:14 +00006 * and others. All Rights Reserved.
7 ************************************************************************************
8 */
9
10#include "unicode/utypes.h"
11
12#if !UCONFIG_NO_BREAK_ITERATION
13
jshin@chromium.org6f31ac32014-03-26 22:15:14 +000014#include "unicode/uchar.h"
15#include "unicode/uniset.h"
16#include "unicode/chariter.h"
17#include "unicode/ures.h"
18#include "unicode/udata.h"
19#include "unicode/putil.h"
20#include "unicode/ustring.h"
21#include "unicode/uscript.h"
22#include "unicode/ucharstrie.h"
23#include "unicode/bytestrie.h"
Jungshik Shinb3189662017-11-07 11:18:34 -080024
25#include "brkeng.h"
26#include "cmemory.h"
27#include "dictbe.h"
Frank Tang3e05d9d2021-11-08 14:04:04 -080028#include "lstmbe.h"
jshin@chromium.org6f31ac32014-03-26 22:15:14 +000029#include "charstr.h"
30#include "dictionarydata.h"
Jungshik Shin70f82502016-01-29 00:32:36 -080031#include "mutex.h"
jshin@chromium.org6f31ac32014-03-26 22:15:14 +000032#include "uvector.h"
33#include "umutex.h"
34#include "uresimp.h"
35#include "ubrkimpl.h"
36
37U_NAMESPACE_BEGIN
38
39/*
40 ******************************************************************
41 */
42
43LanguageBreakEngine::LanguageBreakEngine() {
44}
45
46LanguageBreakEngine::~LanguageBreakEngine() {
47}
48
49/*
50 ******************************************************************
51 */
52
53LanguageBreakFactory::LanguageBreakFactory() {
54}
55
56LanguageBreakFactory::~LanguageBreakFactory() {
57}
58
59/*
60 ******************************************************************
61 */
62
Jungshik Shinf61e46d2018-05-04 13:00:45 -070063UnhandledEngine::UnhandledEngine(UErrorCode &status) : fHandled(nullptr) {
64 (void)status;
jshin@chromium.org6f31ac32014-03-26 22:15:14 +000065}
66
67UnhandledEngine::~UnhandledEngine() {
Jungshik Shinf61e46d2018-05-04 13:00:45 -070068 delete fHandled;
69 fHandled = nullptr;
jshin@chromium.org6f31ac32014-03-26 22:15:14 +000070}
71
72UBool
Jungshik Shinf61e46d2018-05-04 13:00:45 -070073UnhandledEngine::handles(UChar32 c) const {
74 return fHandled && fHandled->contains(c);
jshin@chromium.org6f31ac32014-03-26 22:15:14 +000075}
76
77int32_t
78UnhandledEngine::findBreaks( UText *text,
Jungshik Shinb3189662017-11-07 11:18:34 -080079 int32_t /* startPos */,
80 int32_t endPos,
Frank Tang3e05d9d2021-11-08 14:04:04 -080081 UVector32 &/*foundBreaks*/,
82 UErrorCode &status) const {
83 if (U_FAILURE(status)) return 0;
Jungshik Shinf61e46d2018-05-04 13:00:45 -070084 UChar32 c = utext_current32(text);
85 while((int32_t)utext_getNativeIndex(text) < endPos && fHandled->contains(c)) {
86 utext_next32(text); // TODO: recast loop to work with post-increment operations.
87 c = utext_current32(text);
jshin@chromium.org6f31ac32014-03-26 22:15:14 +000088 }
89 return 0;
90}
91
92void
Jungshik Shinf61e46d2018-05-04 13:00:45 -070093UnhandledEngine::handleCharacter(UChar32 c) {
94 if (fHandled == nullptr) {
95 fHandled = new UnicodeSet();
96 if (fHandled == nullptr) {
97 return;
jshin@chromium.org6f31ac32014-03-26 22:15:14 +000098 }
Jungshik Shinf61e46d2018-05-04 13:00:45 -070099 }
100 if (!fHandled->contains(c)) {
101 UErrorCode status = U_ZERO_ERROR;
102 // Apply the entire script of the character.
103 int32_t script = u_getIntPropertyValue(c, UCHAR_SCRIPT);
104 fHandled->applyIntPropertyValue(UCHAR_SCRIPT, script, status);
jshin@chromium.org6f31ac32014-03-26 22:15:14 +0000105 }
106}
107
108/*
109 ******************************************************************
110 */
111
112ICULanguageBreakFactory::ICULanguageBreakFactory(UErrorCode &/*status*/) {
113 fEngines = 0;
114}
115
116ICULanguageBreakFactory::~ICULanguageBreakFactory() {
117 if (fEngines != 0) {
118 delete fEngines;
119 }
120}
121
122U_NAMESPACE_END
123U_CDECL_BEGIN
124static void U_CALLCONV _deleteEngine(void *obj) {
125 delete (const icu::LanguageBreakEngine *) obj;
126}
127U_CDECL_END
128U_NAMESPACE_BEGIN
129
130const LanguageBreakEngine *
Jungshik Shinf61e46d2018-05-04 13:00:45 -0700131ICULanguageBreakFactory::getEngineFor(UChar32 c) {
jshin@chromium.org6f31ac32014-03-26 22:15:14 +0000132 const LanguageBreakEngine *lbe = NULL;
133 UErrorCode status = U_ZERO_ERROR;
134
Frank Tangb8696612019-10-25 14:58:21 -0700135 static UMutex gBreakEngineMutex;
136 Mutex m(&gBreakEngineMutex);
Jungshik Shin70f82502016-01-29 00:32:36 -0800137
Frank Tang3e05d9d2021-11-08 14:04:04 -0800138 if (fEngines == nullptr) {
139 LocalPointer<UStack> engines(new UStack(_deleteEngine, nullptr, status), status);
140 if (U_FAILURE(status) ) {
Jungshik Shin70f82502016-01-29 00:32:36 -0800141 // Note: no way to return error code to caller.
Frank Tang3e05d9d2021-11-08 14:04:04 -0800142 return nullptr;
Jungshik Shin70f82502016-01-29 00:32:36 -0800143 }
Frank Tang3e05d9d2021-11-08 14:04:04 -0800144 fEngines = engines.orphan();
Jungshik Shin70f82502016-01-29 00:32:36 -0800145 } else {
146 int32_t i = fEngines->size();
jshin@chromium.org6f31ac32014-03-26 22:15:14 +0000147 while (--i >= 0) {
148 lbe = (const LanguageBreakEngine *)(fEngines->elementAt(i));
Jungshik Shinf61e46d2018-05-04 13:00:45 -0700149 if (lbe != NULL && lbe->handles(c)) {
Jungshik Shin70f82502016-01-29 00:32:36 -0800150 return lbe;
jshin@chromium.org6f31ac32014-03-26 22:15:14 +0000151 }
jshin@chromium.org6f31ac32014-03-26 22:15:14 +0000152 }
153 }
jshin@chromium.org6f31ac32014-03-26 22:15:14 +0000154
Jungshik Shin70f82502016-01-29 00:32:36 -0800155 // We didn't find an engine. Create one.
Jungshik Shinf61e46d2018-05-04 13:00:45 -0700156 lbe = loadEngineFor(c);
Frank Tang3e05d9d2021-11-08 14:04:04 -0800157 if (lbe != nullptr) {
Jungshik Shin70f82502016-01-29 00:32:36 -0800158 fEngines->push((void *)lbe, status);
jshin@chromium.org6f31ac32014-03-26 22:15:14 +0000159 }
Frank Tang3e05d9d2021-11-08 14:04:04 -0800160 return U_SUCCESS(status) ? lbe : nullptr;
jshin@chromium.org6f31ac32014-03-26 22:15:14 +0000161}
162
163const LanguageBreakEngine *
Jungshik Shinf61e46d2018-05-04 13:00:45 -0700164ICULanguageBreakFactory::loadEngineFor(UChar32 c) {
jshin@chromium.org6f31ac32014-03-26 22:15:14 +0000165 UErrorCode status = U_ZERO_ERROR;
166 UScriptCode code = uscript_getScript(c, &status);
167 if (U_SUCCESS(status)) {
Frank Tang3e05d9d2021-11-08 14:04:04 -0800168 const LanguageBreakEngine *engine = nullptr;
169 // Try to use LSTM first
170 const LSTMData *data = CreateLSTMDataForScript(code, status);
171 if (U_SUCCESS(status)) {
172 if (data != nullptr) {
173 engine = CreateLSTMBreakEngine(code, data, status);
174 if (U_SUCCESS(status) && engine != nullptr) {
175 return engine;
176 }
177 if (engine != nullptr) {
178 delete engine;
179 engine = nullptr;
180 } else {
181 DeleteLSTMData(data);
182 }
183 }
184 }
185 status = U_ZERO_ERROR; // fallback to dictionary based
Jungshik Shinf61e46d2018-05-04 13:00:45 -0700186 DictionaryMatcher *m = loadDictionaryMatcherFor(code);
jshin@chromium.org6f31ac32014-03-26 22:15:14 +0000187 if (m != NULL) {
jshin@chromium.org6f31ac32014-03-26 22:15:14 +0000188 switch(code) {
189 case USCRIPT_THAI:
190 engine = new ThaiBreakEngine(m, status);
191 break;
192 case USCRIPT_LAO:
193 engine = new LaoBreakEngine(m, status);
194 break;
Jungshik Shin (jungshik at google)0f8746a2015-01-08 15:46:45 -0800195 case USCRIPT_MYANMAR:
196 engine = new BurmeseBreakEngine(m, status);
197 break;
jshin@chromium.org6f31ac32014-03-26 22:15:14 +0000198 case USCRIPT_KHMER:
199 engine = new KhmerBreakEngine(m, status);
200 break;
201
202#if !UCONFIG_NO_NORMALIZATION
203 // CJK not available w/o normalization
204 case USCRIPT_HANGUL:
205 engine = new CjkBreakEngine(m, kKorean, status);
206 break;
207
208 // use same BreakEngine and dictionary for both Chinese and Japanese
209 case USCRIPT_HIRAGANA:
210 case USCRIPT_KATAKANA:
211 case USCRIPT_HAN:
212 engine = new CjkBreakEngine(m, kChineseJapanese, status);
213 break;
214#if 0
215 // TODO: Have to get some characters with script=common handled
216 // by CjkBreakEngine (e.g. U+309B). Simply subjecting
217 // them to CjkBreakEngine does not work. The engine has to
218 // special-case them.
219 case USCRIPT_COMMON:
220 {
221 UBlockCode block = ublock_getCode(code);
222 if (block == UBLOCK_HIRAGANA || block == UBLOCK_KATAKANA)
223 engine = new CjkBreakEngine(dict, kChineseJapanese, status);
224 break;
225 }
226#endif
227#endif
228
229 default:
230 break;
231 }
232 if (engine == NULL) {
233 delete m;
234 }
235 else if (U_FAILURE(status)) {
236 delete engine;
237 engine = NULL;
238 }
239 return engine;
240 }
241 }
242 return NULL;
243}
244
245DictionaryMatcher *
Jungshik Shinf61e46d2018-05-04 13:00:45 -0700246ICULanguageBreakFactory::loadDictionaryMatcherFor(UScriptCode script) {
jshin@chromium.org6f31ac32014-03-26 22:15:14 +0000247 UErrorCode status = U_ZERO_ERROR;
248 // open root from brkitr tree.
249 UResourceBundle *b = ures_open(U_ICUDATA_BRKITR, "", &status);
250 b = ures_getByKeyWithFallback(b, "dictionaries", b, &status);
251 int32_t dictnlength = 0;
252 const UChar *dictfname =
253 ures_getStringByKeyWithFallback(b, uscript_getShortName(script), &dictnlength, &status);
254 if (U_FAILURE(status)) {
255 ures_close(b);
256 return NULL;
257 }
258 CharString dictnbuf;
259 CharString ext;
260 const UChar *extStart = u_memrchr(dictfname, 0x002e, dictnlength); // last dot
261 if (extStart != NULL) {
262 int32_t len = (int32_t)(extStart - dictfname);
263 ext.appendInvariantChars(UnicodeString(FALSE, extStart + 1, dictnlength - len - 1), status);
264 dictnlength = len;
265 }
266 dictnbuf.appendInvariantChars(UnicodeString(FALSE, dictfname, dictnlength), status);
267 ures_close(b);
268
269 UDataMemory *file = udata_open(U_ICUDATA_BRKITR, ext.data(), dictnbuf.data(), &status);
270 if (U_SUCCESS(status)) {
271 // build trie
272 const uint8_t *data = (const uint8_t *)udata_getMemory(file);
273 const int32_t *indexes = (const int32_t *)data;
274 const int32_t offset = indexes[DictionaryData::IX_STRING_TRIE_OFFSET];
275 const int32_t trieType = indexes[DictionaryData::IX_TRIE_TYPE] & DictionaryData::TRIE_TYPE_MASK;
276 DictionaryMatcher *m = NULL;
277 if (trieType == DictionaryData::TRIE_TYPE_BYTES) {
278 const int32_t transform = indexes[DictionaryData::IX_TRANSFORM];
279 const char *characters = (const char *)(data + offset);
280 m = new BytesDictionaryMatcher(characters, transform, file);
281 }
282 else if (trieType == DictionaryData::TRIE_TYPE_UCHARS) {
283 const UChar *characters = (const UChar *)(data + offset);
284 m = new UCharsDictionaryMatcher(characters, file);
285 }
286 if (m == NULL) {
287 // no matcher exists to take ownership - either we are an invalid
288 // type or memory allocation failed
289 udata_close(file);
290 }
291 return m;
292 } else if (dictfname != NULL) {
293 // we don't have a dictionary matcher.
294 // returning NULL here will cause us to fail to find a dictionary break engine, as expected
295 status = U_ZERO_ERROR;
296 return NULL;
297 }
298 return NULL;
299}
300
301U_NAMESPACE_END
302
303#endif /* #if !UCONFIG_NO_BREAK_ITERATION */