usprep.cpp 27 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872
  1. // © 2016 and later: Unicode, Inc. and others.
  2. // License & terms of use: http://www.unicode.org/copyright.html
  3. /*
  4. *******************************************************************************
  5. *
  6. * Copyright (C) 2003-2016, International Business Machines
  7. * Corporation and others. All Rights Reserved.
  8. *
  9. *******************************************************************************
  10. * file name: usprep.cpp
  11. * encoding: UTF-8
  12. * tab size: 8 (not used)
  13. * indentation:4
  14. *
  15. * created on: 2003jul2
  16. * created by: Ram Viswanadha
  17. */
  18. #include "unicode/utypes.h"
  19. #if !UCONFIG_NO_IDNA
  20. #include "unicode/usprep.h"
  21. #include "unicode/normalizer2.h"
  22. #include "unicode/ustring.h"
  23. #include "unicode/uchar.h"
  24. #include "unicode/uversion.h"
  25. #include "umutex.h"
  26. #include "cmemory.h"
  27. #include "sprpimpl.h"
  28. #include "ustr_imp.h"
  29. #include "uhash.h"
  30. #include "cstring.h"
  31. #include "udataswp.h"
  32. #include "ucln_cmn.h"
  33. #include "ubidi_props.h"
  34. #include "uprops.h"
  35. U_NAMESPACE_USE
  36. U_CDECL_BEGIN
  37. /*
  38. Static cache for already opened StringPrep profiles
  39. */
  40. static UHashtable *SHARED_DATA_HASHTABLE = nullptr;
  41. static icu::UInitOnce gSharedDataInitOnce {};
  42. static UMutex usprepMutex;
  43. /* format version of spp file */
  44. //static uint8_t formatVersion[4]={ 0, 0, 0, 0 };
  45. /* the Unicode version of the sprep data */
  46. static UVersionInfo dataVersion={ 0, 0, 0, 0 };
  47. /* Profile names must be aligned to UStringPrepProfileType */
  48. static const char * const PROFILE_NAMES[] = {
  49. "rfc3491", /* USPREP_RFC3491_NAMEPREP */
  50. "rfc3530cs", /* USPREP_RFC3530_NFS4_CS_PREP */
  51. "rfc3530csci", /* USPREP_RFC3530_NFS4_CS_PREP_CI */
  52. "rfc3491", /* USPREP_RFC3530_NSF4_CIS_PREP */
  53. "rfc3530mixp", /* USPREP_RFC3530_NSF4_MIXED_PREP_PREFIX */
  54. "rfc3491", /* USPREP_RFC3530_NSF4_MIXED_PREP_SUFFIX */
  55. "rfc3722", /* USPREP_RFC3722_ISCSI */
  56. "rfc3920node", /* USPREP_RFC3920_NODEPREP */
  57. "rfc3920res", /* USPREP_RFC3920_RESOURCEPREP */
  58. "rfc4011", /* USPREP_RFC4011_MIB */
  59. "rfc4013", /* USPREP_RFC4013_SASLPREP */
  60. "rfc4505", /* USPREP_RFC4505_TRACE */
  61. "rfc4518", /* USPREP_RFC4518_LDAP */
  62. "rfc4518ci", /* USPREP_RFC4518_LDAP_CI */
  63. };
  64. static UBool U_CALLCONV
  65. isSPrepAcceptable(void * /* context */,
  66. const char * /* type */,
  67. const char * /* name */,
  68. const UDataInfo *pInfo) {
  69. if(
  70. pInfo->size>=20 &&
  71. pInfo->isBigEndian==U_IS_BIG_ENDIAN &&
  72. pInfo->charsetFamily==U_CHARSET_FAMILY &&
  73. pInfo->dataFormat[0]==0x53 && /* dataFormat="SPRP" */
  74. pInfo->dataFormat[1]==0x50 &&
  75. pInfo->dataFormat[2]==0x52 &&
  76. pInfo->dataFormat[3]==0x50 &&
  77. pInfo->formatVersion[0]==3 &&
  78. pInfo->formatVersion[2]==UTRIE_SHIFT &&
  79. pInfo->formatVersion[3]==UTRIE_INDEX_SHIFT
  80. ) {
  81. //uprv_memcpy(formatVersion, pInfo->formatVersion, 4);
  82. uprv_memcpy(dataVersion, pInfo->dataVersion, 4);
  83. return true;
  84. } else {
  85. return false;
  86. }
  87. }
  88. static int32_t U_CALLCONV
  89. getSPrepFoldingOffset(uint32_t data) {
  90. return (int32_t)data;
  91. }
  92. /* hashes an entry */
  93. static int32_t U_CALLCONV
  94. hashEntry(const UHashTok parm) {
  95. UStringPrepKey *b = (UStringPrepKey *)parm.pointer;
  96. UHashTok namekey, pathkey;
  97. namekey.pointer = b->name;
  98. pathkey.pointer = b->path;
  99. uint32_t unsignedHash = static_cast<uint32_t>(uhash_hashChars(namekey)) +
  100. 37u * static_cast<uint32_t>(uhash_hashChars(pathkey));
  101. return static_cast<int32_t>(unsignedHash);
  102. }
  103. /* compares two entries */
  104. static UBool U_CALLCONV
  105. compareEntries(const UHashTok p1, const UHashTok p2) {
  106. UStringPrepKey *b1 = (UStringPrepKey *)p1.pointer;
  107. UStringPrepKey *b2 = (UStringPrepKey *)p2.pointer;
  108. UHashTok name1, name2, path1, path2;
  109. name1.pointer = b1->name;
  110. name2.pointer = b2->name;
  111. path1.pointer = b1->path;
  112. path2.pointer = b2->path;
  113. return ((UBool)(uhash_compareChars(name1, name2) &
  114. uhash_compareChars(path1, path2)));
  115. }
  116. static void
  117. usprep_unload(UStringPrepProfile* data){
  118. udata_close(data->sprepData);
  119. }
  120. static int32_t
  121. usprep_internal_flushCache(UBool noRefCount){
  122. UStringPrepProfile *profile = nullptr;
  123. UStringPrepKey *key = nullptr;
  124. int32_t pos = UHASH_FIRST;
  125. int32_t deletedNum = 0;
  126. const UHashElement *e;
  127. /*
  128. * if shared data hasn't even been lazy evaluated yet
  129. * return 0
  130. */
  131. umtx_lock(&usprepMutex);
  132. if (SHARED_DATA_HASHTABLE == nullptr) {
  133. umtx_unlock(&usprepMutex);
  134. return 0;
  135. }
  136. /*creates an enumeration to iterate through every element in the table */
  137. while ((e = uhash_nextElement(SHARED_DATA_HASHTABLE, &pos)) != nullptr)
  138. {
  139. profile = (UStringPrepProfile *) e->value.pointer;
  140. key = (UStringPrepKey *) e->key.pointer;
  141. if ((noRefCount== false && profile->refCount == 0) ||
  142. noRefCount) {
  143. deletedNum++;
  144. uhash_removeElement(SHARED_DATA_HASHTABLE, e);
  145. /* unload the data */
  146. usprep_unload(profile);
  147. if(key->name != nullptr) {
  148. uprv_free(key->name);
  149. key->name=nullptr;
  150. }
  151. if(key->path != nullptr) {
  152. uprv_free(key->path);
  153. key->path=nullptr;
  154. }
  155. uprv_free(profile);
  156. uprv_free(key);
  157. }
  158. }
  159. umtx_unlock(&usprepMutex);
  160. return deletedNum;
  161. }
  162. /* Works just like ucnv_flushCache()
  163. static int32_t
  164. usprep_flushCache(){
  165. return usprep_internal_flushCache(false);
  166. }
  167. */
  168. static UBool U_CALLCONV usprep_cleanup(){
  169. if (SHARED_DATA_HASHTABLE != nullptr) {
  170. usprep_internal_flushCache(true);
  171. if (SHARED_DATA_HASHTABLE != nullptr && uhash_count(SHARED_DATA_HASHTABLE) == 0) {
  172. uhash_close(SHARED_DATA_HASHTABLE);
  173. SHARED_DATA_HASHTABLE = nullptr;
  174. }
  175. }
  176. gSharedDataInitOnce.reset();
  177. return (SHARED_DATA_HASHTABLE == nullptr);
  178. }
  179. U_CDECL_END
  180. /** Initializes the cache for resources */
  181. static void U_CALLCONV
  182. createCache(UErrorCode &status) {
  183. SHARED_DATA_HASHTABLE = uhash_open(hashEntry, compareEntries, nullptr, &status);
  184. if (U_FAILURE(status)) {
  185. SHARED_DATA_HASHTABLE = nullptr;
  186. }
  187. ucln_common_registerCleanup(UCLN_COMMON_USPREP, usprep_cleanup);
  188. }
  189. static void
  190. initCache(UErrorCode *status) {
  191. umtx_initOnce(gSharedDataInitOnce, &createCache, *status);
  192. }
  193. static UBool U_CALLCONV
  194. loadData(UStringPrepProfile* profile,
  195. const char* path,
  196. const char* name,
  197. const char* type,
  198. UErrorCode* errorCode) {
  199. /* load Unicode SPREP data from file */
  200. UTrie _sprepTrie={ 0,0,0,0,0,0,0 };
  201. UDataMemory *dataMemory;
  202. const int32_t *p=nullptr;
  203. const uint8_t *pb;
  204. UVersionInfo normUnicodeVersion;
  205. int32_t normUniVer, sprepUniVer, normCorrVer;
  206. if(errorCode==nullptr || U_FAILURE(*errorCode)) {
  207. return 0;
  208. }
  209. /* open the data outside the mutex block */
  210. //TODO: change the path
  211. dataMemory=udata_openChoice(path, type, name, isSPrepAcceptable, nullptr, errorCode);
  212. if(U_FAILURE(*errorCode)) {
  213. return false;
  214. }
  215. p=(const int32_t *)udata_getMemory(dataMemory);
  216. pb=(const uint8_t *)(p+_SPREP_INDEX_TOP);
  217. utrie_unserialize(&_sprepTrie, pb, p[_SPREP_INDEX_TRIE_SIZE], errorCode);
  218. _sprepTrie.getFoldingOffset=getSPrepFoldingOffset;
  219. if(U_FAILURE(*errorCode)) {
  220. udata_close(dataMemory);
  221. return false;
  222. }
  223. /* in the mutex block, set the data for this process */
  224. umtx_lock(&usprepMutex);
  225. if(profile->sprepData==nullptr) {
  226. profile->sprepData=dataMemory;
  227. dataMemory=nullptr;
  228. uprv_memcpy(&profile->indexes, p, sizeof(profile->indexes));
  229. uprv_memcpy(&profile->sprepTrie, &_sprepTrie, sizeof(UTrie));
  230. } else {
  231. p=(const int32_t *)udata_getMemory(profile->sprepData);
  232. }
  233. umtx_unlock(&usprepMutex);
  234. /* initialize some variables */
  235. profile->mappingData=(uint16_t *)((uint8_t *)(p+_SPREP_INDEX_TOP)+profile->indexes[_SPREP_INDEX_TRIE_SIZE]);
  236. u_getUnicodeVersion(normUnicodeVersion);
  237. normUniVer = (normUnicodeVersion[0] << 24) + (normUnicodeVersion[1] << 16) +
  238. (normUnicodeVersion[2] << 8 ) + (normUnicodeVersion[3]);
  239. sprepUniVer = (dataVersion[0] << 24) + (dataVersion[1] << 16) +
  240. (dataVersion[2] << 8 ) + (dataVersion[3]);
  241. normCorrVer = profile->indexes[_SPREP_NORM_CORRECTNS_LAST_UNI_VERSION];
  242. if(U_FAILURE(*errorCode)){
  243. udata_close(dataMemory);
  244. return false;
  245. }
  246. if( normUniVer < sprepUniVer && /* the Unicode version of SPREP file must be less than the Unicode Version of the normalization data */
  247. normUniVer < normCorrVer && /* the Unicode version of the NormalizationCorrections.txt file should be less than the Unicode Version of the normalization data */
  248. ((profile->indexes[_SPREP_OPTIONS] & _SPREP_NORMALIZATION_ON) > 0) /* normalization turned on*/
  249. ){
  250. *errorCode = U_INVALID_FORMAT_ERROR;
  251. udata_close(dataMemory);
  252. return false;
  253. }
  254. profile->isDataLoaded = true;
  255. /* if a different thread set it first, then close the extra data */
  256. if(dataMemory!=nullptr) {
  257. udata_close(dataMemory); /* nullptr if it was set correctly */
  258. }
  259. return profile->isDataLoaded;
  260. }
  261. static UStringPrepProfile*
  262. usprep_getProfile(const char* path,
  263. const char* name,
  264. UErrorCode *status){
  265. UStringPrepProfile* profile = nullptr;
  266. initCache(status);
  267. if(U_FAILURE(*status)){
  268. return nullptr;
  269. }
  270. UStringPrepKey stackKey;
  271. /*
  272. * const is cast way to save malloc, strcpy and free calls
  273. * we use the passed in pointers for fetching the data from the
  274. * hash table which is safe
  275. */
  276. stackKey.name = (char*) name;
  277. stackKey.path = (char*) path;
  278. /* fetch the data from the cache */
  279. umtx_lock(&usprepMutex);
  280. profile = (UStringPrepProfile*) (uhash_get(SHARED_DATA_HASHTABLE,&stackKey));
  281. if(profile != nullptr) {
  282. profile->refCount++;
  283. }
  284. umtx_unlock(&usprepMutex);
  285. if(profile == nullptr) {
  286. /* else load the data and put the data in the cache */
  287. LocalMemory<UStringPrepProfile> newProfile;
  288. if(newProfile.allocateInsteadAndReset() == nullptr) {
  289. *status = U_MEMORY_ALLOCATION_ERROR;
  290. return nullptr;
  291. }
  292. /* load the data */
  293. if(!loadData(newProfile.getAlias(), path, name, _SPREP_DATA_TYPE, status) || U_FAILURE(*status) ){
  294. return nullptr;
  295. }
  296. /* get the options */
  297. newProfile->doNFKC = (UBool)((newProfile->indexes[_SPREP_OPTIONS] & _SPREP_NORMALIZATION_ON) > 0);
  298. newProfile->checkBiDi = (UBool)((newProfile->indexes[_SPREP_OPTIONS] & _SPREP_CHECK_BIDI_ON) > 0);
  299. LocalMemory<UStringPrepKey> key;
  300. LocalMemory<char> keyName;
  301. LocalMemory<char> keyPath;
  302. if( key.allocateInsteadAndReset() == nullptr ||
  303. keyName.allocateInsteadAndCopy(static_cast<int32_t>(uprv_strlen(name)+1)) == nullptr ||
  304. (path != nullptr &&
  305. keyPath.allocateInsteadAndCopy(static_cast<int32_t>(uprv_strlen(path)+1)) == nullptr)
  306. ) {
  307. *status = U_MEMORY_ALLOCATION_ERROR;
  308. usprep_unload(newProfile.getAlias());
  309. return nullptr;
  310. }
  311. umtx_lock(&usprepMutex);
  312. // If another thread already inserted the same key/value, refcount and cleanup our thread data
  313. profile = (UStringPrepProfile*) (uhash_get(SHARED_DATA_HASHTABLE,&stackKey));
  314. if(profile != nullptr) {
  315. profile->refCount++;
  316. usprep_unload(newProfile.getAlias());
  317. }
  318. else {
  319. /* initialize the key members */
  320. key->name = keyName.orphan();
  321. uprv_strcpy(key->name, name);
  322. if(path != nullptr){
  323. key->path = keyPath.orphan();
  324. uprv_strcpy(key->path, path);
  325. }
  326. profile = newProfile.orphan();
  327. /* add the data object to the cache */
  328. profile->refCount = 1;
  329. uhash_put(SHARED_DATA_HASHTABLE, key.orphan(), profile, status);
  330. }
  331. umtx_unlock(&usprepMutex);
  332. }
  333. return profile;
  334. }
  335. U_CAPI UStringPrepProfile* U_EXPORT2
  336. usprep_open(const char* path,
  337. const char* name,
  338. UErrorCode* status){
  339. if(status == nullptr || U_FAILURE(*status)){
  340. return nullptr;
  341. }
  342. /* initialize the profile struct members */
  343. return usprep_getProfile(path,name,status);
  344. }
  345. U_CAPI UStringPrepProfile* U_EXPORT2
  346. usprep_openByType(UStringPrepProfileType type,
  347. UErrorCode* status) {
  348. if(status == nullptr || U_FAILURE(*status)){
  349. return nullptr;
  350. }
  351. int32_t index = (int32_t)type;
  352. if (index < 0 || index >= UPRV_LENGTHOF(PROFILE_NAMES)) {
  353. *status = U_ILLEGAL_ARGUMENT_ERROR;
  354. return nullptr;
  355. }
  356. return usprep_open(nullptr, PROFILE_NAMES[index], status);
  357. }
  358. U_CAPI void U_EXPORT2
  359. usprep_close(UStringPrepProfile* profile){
  360. if(profile==nullptr){
  361. return;
  362. }
  363. umtx_lock(&usprepMutex);
  364. /* decrement the ref count*/
  365. if(profile->refCount > 0){
  366. profile->refCount--;
  367. }
  368. umtx_unlock(&usprepMutex);
  369. }
  370. U_CFUNC void
  371. uprv_syntaxError(const char16_t* rules,
  372. int32_t pos,
  373. int32_t rulesLen,
  374. UParseError* parseError){
  375. if(parseError == nullptr){
  376. return;
  377. }
  378. parseError->offset = pos;
  379. parseError->line = 0 ; // we are not using line numbers
  380. // for pre-context
  381. int32_t start = (pos < U_PARSE_CONTEXT_LEN)? 0 : (pos - (U_PARSE_CONTEXT_LEN-1));
  382. int32_t limit = pos;
  383. u_memcpy(parseError->preContext,rules+start,limit-start);
  384. //null terminate the buffer
  385. parseError->preContext[limit-start] = 0;
  386. // for post-context; include error rules[pos]
  387. start = pos;
  388. limit = start + (U_PARSE_CONTEXT_LEN-1);
  389. if (limit > rulesLen) {
  390. limit = rulesLen;
  391. }
  392. if (start < rulesLen) {
  393. u_memcpy(parseError->postContext,rules+start,limit-start);
  394. }
  395. //null terminate the buffer
  396. parseError->postContext[limit-start]= 0;
  397. }
  398. static inline UStringPrepType
  399. getValues(uint16_t trieWord, int16_t& value, UBool& isIndex){
  400. UStringPrepType type;
  401. if(trieWord == 0){
  402. /*
  403. * Initial value stored in the mapping table
  404. * just return USPREP_TYPE_LIMIT .. so that
  405. * the source codepoint is copied to the destination
  406. */
  407. type = USPREP_TYPE_LIMIT;
  408. isIndex =false;
  409. value = 0;
  410. }else if(trieWord >= _SPREP_TYPE_THRESHOLD){
  411. type = (UStringPrepType) (trieWord - _SPREP_TYPE_THRESHOLD);
  412. isIndex =false;
  413. value = 0;
  414. }else{
  415. /* get the type */
  416. type = USPREP_MAP;
  417. /* ascertain if the value is index or delta */
  418. if(trieWord & 0x02){
  419. isIndex = true;
  420. value = trieWord >> 2; //mask off the lower 2 bits and shift
  421. }else{
  422. isIndex = false;
  423. value = (int16_t)trieWord;
  424. value = (value >> 2);
  425. }
  426. if((trieWord>>2) == _SPREP_MAX_INDEX_VALUE){
  427. type = USPREP_DELETE;
  428. isIndex =false;
  429. value = 0;
  430. }
  431. }
  432. return type;
  433. }
  434. // TODO: change to writing to UnicodeString not char16_t *
  435. static int32_t
  436. usprep_map( const UStringPrepProfile* profile,
  437. const char16_t* src, int32_t srcLength,
  438. char16_t* dest, int32_t destCapacity,
  439. int32_t options,
  440. UParseError* parseError,
  441. UErrorCode* status ){
  442. uint16_t result;
  443. int32_t destIndex=0;
  444. int32_t srcIndex;
  445. UBool allowUnassigned = (UBool) ((options & USPREP_ALLOW_UNASSIGNED)>0);
  446. UStringPrepType type;
  447. int16_t value;
  448. UBool isIndex;
  449. const int32_t* indexes = profile->indexes;
  450. // no error checking the caller check for error and arguments
  451. // no string length check the caller finds out the string length
  452. for(srcIndex=0;srcIndex<srcLength;){
  453. UChar32 ch;
  454. U16_NEXT(src,srcIndex,srcLength,ch);
  455. result=0;
  456. UTRIE_GET16(&profile->sprepTrie,ch,result);
  457. type = getValues(result, value, isIndex);
  458. // check if the source codepoint is unassigned
  459. if(type == USPREP_UNASSIGNED && allowUnassigned == false){
  460. uprv_syntaxError(src,srcIndex-U16_LENGTH(ch), srcLength,parseError);
  461. *status = U_STRINGPREP_UNASSIGNED_ERROR;
  462. return 0;
  463. }else if(type == USPREP_MAP){
  464. int32_t index, length;
  465. if(isIndex){
  466. index = value;
  467. if(index >= indexes[_SPREP_ONE_UCHAR_MAPPING_INDEX_START] &&
  468. index < indexes[_SPREP_TWO_UCHARS_MAPPING_INDEX_START]){
  469. length = 1;
  470. }else if(index >= indexes[_SPREP_TWO_UCHARS_MAPPING_INDEX_START] &&
  471. index < indexes[_SPREP_THREE_UCHARS_MAPPING_INDEX_START]){
  472. length = 2;
  473. }else if(index >= indexes[_SPREP_THREE_UCHARS_MAPPING_INDEX_START] &&
  474. index < indexes[_SPREP_FOUR_UCHARS_MAPPING_INDEX_START]){
  475. length = 3;
  476. }else{
  477. length = profile->mappingData[index++];
  478. }
  479. /* copy mapping to destination */
  480. for(int32_t i=0; i< length; i++){
  481. if(destIndex < destCapacity ){
  482. dest[destIndex] = profile->mappingData[index+i];
  483. }
  484. destIndex++; /* for pre-flighting */
  485. }
  486. continue;
  487. }else{
  488. // subtract the delta to arrive at the code point
  489. ch -= value;
  490. }
  491. }else if(type==USPREP_DELETE){
  492. // just consume the codepoint and continue
  493. continue;
  494. }
  495. //copy the code point into destination
  496. if(ch <= 0xFFFF){
  497. if(destIndex < destCapacity ){
  498. dest[destIndex] = (char16_t)ch;
  499. }
  500. destIndex++;
  501. }else{
  502. if(destIndex+1 < destCapacity ){
  503. dest[destIndex] = U16_LEAD(ch);
  504. dest[destIndex+1] = U16_TRAIL(ch);
  505. }
  506. destIndex +=2;
  507. }
  508. }
  509. return u_terminateUChars(dest, destCapacity, destIndex, status);
  510. }
  511. /*
  512. 1) Map -- For each character in the input, check if it has a mapping
  513. and, if so, replace it with its mapping.
  514. 2) Normalize -- Possibly normalize the result of step 1 using Unicode
  515. normalization.
  516. 3) Prohibit -- Check for any characters that are not allowed in the
  517. output. If any are found, return an error.
  518. 4) Check bidi -- Possibly check for right-to-left characters, and if
  519. any are found, make sure that the whole string satisfies the
  520. requirements for bidirectional strings. If the string does not
  521. satisfy the requirements for bidirectional strings, return an
  522. error.
  523. [Unicode3.2] defines several bidirectional categories; each character
  524. has one bidirectional category assigned to it. For the purposes of
  525. the requirements below, an "RandALCat character" is a character that
  526. has Unicode bidirectional categories "R" or "AL"; an "LCat character"
  527. is a character that has Unicode bidirectional category "L". Note
  528. that there are many characters which fall in neither of the above
  529. definitions; Latin digits (<U+0030> through <U+0039>) are examples of
  530. this because they have bidirectional category "EN".
  531. In any profile that specifies bidirectional character handling, all
  532. three of the following requirements MUST be met:
  533. 1) The characters in section 5.8 MUST be prohibited.
  534. 2) If a string contains any RandALCat character, the string MUST NOT
  535. contain any LCat character.
  536. 3) If a string contains any RandALCat character, a RandALCat
  537. character MUST be the first character of the string, and a
  538. RandALCat character MUST be the last character of the string.
  539. */
  540. U_CAPI int32_t U_EXPORT2
  541. usprep_prepare( const UStringPrepProfile* profile,
  542. const char16_t* src, int32_t srcLength,
  543. char16_t* dest, int32_t destCapacity,
  544. int32_t options,
  545. UParseError* parseError,
  546. UErrorCode* status ){
  547. // check error status
  548. if(U_FAILURE(*status)){
  549. return 0;
  550. }
  551. //check arguments
  552. if(profile==nullptr ||
  553. (src==nullptr ? srcLength!=0 : srcLength<-1) ||
  554. (dest==nullptr ? destCapacity!=0 : destCapacity<0)) {
  555. *status=U_ILLEGAL_ARGUMENT_ERROR;
  556. return 0;
  557. }
  558. //get the string length
  559. if(srcLength < 0){
  560. srcLength = u_strlen(src);
  561. }
  562. // map
  563. UnicodeString s1;
  564. char16_t *b1 = s1.getBuffer(srcLength);
  565. if(b1==nullptr){
  566. *status = U_MEMORY_ALLOCATION_ERROR;
  567. return 0;
  568. }
  569. int32_t b1Len = usprep_map(profile, src, srcLength,
  570. b1, s1.getCapacity(), options, parseError, status);
  571. s1.releaseBuffer(U_SUCCESS(*status) ? b1Len : 0);
  572. if(*status == U_BUFFER_OVERFLOW_ERROR){
  573. // redo processing of string
  574. /* we do not have enough room so grow the buffer*/
  575. b1 = s1.getBuffer(b1Len);
  576. if(b1==nullptr){
  577. *status = U_MEMORY_ALLOCATION_ERROR;
  578. return 0;
  579. }
  580. *status = U_ZERO_ERROR; // reset error
  581. b1Len = usprep_map(profile, src, srcLength,
  582. b1, s1.getCapacity(), options, parseError, status);
  583. s1.releaseBuffer(U_SUCCESS(*status) ? b1Len : 0);
  584. }
  585. if(U_FAILURE(*status)){
  586. return 0;
  587. }
  588. // normalize
  589. UnicodeString s2;
  590. if(profile->doNFKC){
  591. const Normalizer2 *n2 = Normalizer2::getNFKCInstance(*status);
  592. FilteredNormalizer2 fn2(*n2, *uniset_getUnicode32Instance(*status));
  593. if(U_FAILURE(*status)){
  594. return 0;
  595. }
  596. fn2.normalize(s1, s2, *status);
  597. }else{
  598. s2.fastCopyFrom(s1);
  599. }
  600. if(U_FAILURE(*status)){
  601. return 0;
  602. }
  603. // Prohibit and checkBiDi in one pass
  604. const char16_t *b2 = s2.getBuffer();
  605. int32_t b2Len = s2.length();
  606. UCharDirection direction=U_CHAR_DIRECTION_COUNT, firstCharDir=U_CHAR_DIRECTION_COUNT;
  607. UBool leftToRight=false, rightToLeft=false;
  608. int32_t rtlPos =-1, ltrPos =-1;
  609. for(int32_t b2Index=0; b2Index<b2Len;){
  610. UChar32 ch = 0;
  611. U16_NEXT(b2, b2Index, b2Len, ch);
  612. uint16_t result;
  613. UTRIE_GET16(&profile->sprepTrie,ch,result);
  614. int16_t value;
  615. UBool isIndex;
  616. UStringPrepType type = getValues(result, value, isIndex);
  617. if( type == USPREP_PROHIBITED ||
  618. ((result < _SPREP_TYPE_THRESHOLD) && (result & 0x01) /* first bit says it the code point is prohibited*/)
  619. ){
  620. *status = U_STRINGPREP_PROHIBITED_ERROR;
  621. uprv_syntaxError(b2, b2Index-U16_LENGTH(ch), b2Len, parseError);
  622. return 0;
  623. }
  624. if(profile->checkBiDi) {
  625. direction = ubidi_getClass(ch);
  626. if(firstCharDir == U_CHAR_DIRECTION_COUNT){
  627. firstCharDir = direction;
  628. }
  629. if(direction == U_LEFT_TO_RIGHT){
  630. leftToRight = true;
  631. ltrPos = b2Index-1;
  632. }
  633. if(direction == U_RIGHT_TO_LEFT || direction == U_RIGHT_TO_LEFT_ARABIC){
  634. rightToLeft = true;
  635. rtlPos = b2Index-1;
  636. }
  637. }
  638. }
  639. if(profile->checkBiDi){
  640. // satisfy 2
  641. if( leftToRight && rightToLeft){
  642. *status = U_STRINGPREP_CHECK_BIDI_ERROR;
  643. uprv_syntaxError(b2,(rtlPos>ltrPos) ? rtlPos : ltrPos, b2Len, parseError);
  644. return 0;
  645. }
  646. //satisfy 3
  647. if( rightToLeft &&
  648. !((firstCharDir == U_RIGHT_TO_LEFT || firstCharDir == U_RIGHT_TO_LEFT_ARABIC) &&
  649. (direction == U_RIGHT_TO_LEFT || direction == U_RIGHT_TO_LEFT_ARABIC))
  650. ){
  651. *status = U_STRINGPREP_CHECK_BIDI_ERROR;
  652. uprv_syntaxError(b2, rtlPos, b2Len, parseError);
  653. return false;
  654. }
  655. }
  656. return s2.extract(dest, destCapacity, *status);
  657. }
  658. /* data swapping ------------------------------------------------------------ */
  659. U_CAPI int32_t U_EXPORT2
  660. usprep_swap(const UDataSwapper *ds,
  661. const void *inData, int32_t length, void *outData,
  662. UErrorCode *pErrorCode) {
  663. const UDataInfo *pInfo;
  664. int32_t headerSize;
  665. const uint8_t *inBytes;
  666. uint8_t *outBytes;
  667. const int32_t *inIndexes;
  668. int32_t indexes[16];
  669. int32_t i, offset, count, size;
  670. /* udata_swapDataHeader checks the arguments */
  671. headerSize=udata_swapDataHeader(ds, inData, length, outData, pErrorCode);
  672. if(pErrorCode==nullptr || U_FAILURE(*pErrorCode)) {
  673. return 0;
  674. }
  675. /* check data format and format version */
  676. pInfo=(const UDataInfo *)((const char *)inData+4);
  677. if(!(
  678. pInfo->dataFormat[0]==0x53 && /* dataFormat="SPRP" */
  679. pInfo->dataFormat[1]==0x50 &&
  680. pInfo->dataFormat[2]==0x52 &&
  681. pInfo->dataFormat[3]==0x50 &&
  682. pInfo->formatVersion[0]==3
  683. )) {
  684. udata_printError(ds, "usprep_swap(): data format %02x.%02x.%02x.%02x (format version %02x) is not recognized as StringPrep .spp data\n",
  685. pInfo->dataFormat[0], pInfo->dataFormat[1],
  686. pInfo->dataFormat[2], pInfo->dataFormat[3],
  687. pInfo->formatVersion[0]);
  688. *pErrorCode=U_UNSUPPORTED_ERROR;
  689. return 0;
  690. }
  691. inBytes=(const uint8_t *)inData+headerSize;
  692. outBytes= (outData == nullptr ) ? nullptr : (uint8_t *)outData+headerSize;
  693. inIndexes=(const int32_t *)inBytes;
  694. if(length>=0) {
  695. length-=headerSize;
  696. if(length<16*4) {
  697. udata_printError(ds, "usprep_swap(): too few bytes (%d after header) for StringPrep .spp data\n",
  698. length);
  699. *pErrorCode=U_INDEX_OUTOFBOUNDS_ERROR;
  700. return 0;
  701. }
  702. }
  703. /* read the first 16 indexes (ICU 2.8/format version 3: _SPREP_INDEX_TOP==16, might grow) */
  704. for(i=0; i<16; ++i) {
  705. indexes[i]=udata_readInt32(ds, inIndexes[i]);
  706. }
  707. /* calculate the total length of the data */
  708. size=
  709. 16*4+ /* size of indexes[] */
  710. indexes[_SPREP_INDEX_TRIE_SIZE]+
  711. indexes[_SPREP_INDEX_MAPPING_DATA_SIZE];
  712. if(length>=0) {
  713. if(length<size) {
  714. udata_printError(ds, "usprep_swap(): too few bytes (%d after header) for all of StringPrep .spp data\n",
  715. length);
  716. *pErrorCode=U_INDEX_OUTOFBOUNDS_ERROR;
  717. return 0;
  718. }
  719. /* copy the data for inaccessible bytes */
  720. if(inBytes!=outBytes) {
  721. uprv_memcpy(outBytes, inBytes, size);
  722. }
  723. offset=0;
  724. /* swap the int32_t indexes[] */
  725. count=16*4;
  726. ds->swapArray32(ds, inBytes, count, outBytes, pErrorCode);
  727. offset+=count;
  728. /* swap the UTrie */
  729. count=indexes[_SPREP_INDEX_TRIE_SIZE];
  730. utrie_swap(ds, inBytes+offset, count, outBytes+offset, pErrorCode);
  731. offset+=count;
  732. /* swap the uint16_t mappingTable[] */
  733. count=indexes[_SPREP_INDEX_MAPPING_DATA_SIZE];
  734. ds->swapArray16(ds, inBytes+offset, count, outBytes+offset, pErrorCode);
  735. //offset+=count;
  736. }
  737. return headerSize+size;
  738. }
  739. #endif /* #if !UCONFIG_NO_IDNA */