make_unicode 90 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843
  1. #!/usr/bin/perl -w
  2. #
  3. # Generate code page .c files from ftp.unicode.org descriptions
  4. #
  5. # Copyright 2000 Alexandre Julliard
  6. #
  7. # This library is free software; you can redistribute it and/or
  8. # modify it under the terms of the GNU Lesser General Public
  9. # License as published by the Free Software Foundation; either
  10. # version 2.1 of the License, or (at your option) any later version.
  11. #
  12. # This library is distributed in the hope that it will be useful,
  13. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  15. # Lesser General Public License for more details.
  16. #
  17. # You should have received a copy of the GNU Lesser General Public
  18. # License along with this library; if not, write to the Free Software
  19. # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA
  20. #
  21. use strict;
  22. # base URLs for www.unicode.org files
  23. my $UNIVERSION = "13.0.0";
  24. my $UNIDATA = "https://www.unicode.org/Public/$UNIVERSION/ucd/UCD.zip";
  25. my $IDNADATA = "https://www.unicode.org/Public/idna/$UNIVERSION";
  26. my $JISDATA = "https://www.unicode.org/Public/MAPPINGS/OBSOLETE/EASTASIA/JIS";
  27. my $KSCDATA = "https://www.unicode.org/Public/MAPPINGS/OBSOLETE/EASTASIA/KSC";
  28. my $REPORTS = "http://www.unicode.org/reports";
  29. my $MSDATA = "https://download.microsoft.com/download/C/F/7/CF713A5E-9FBC-4FD6-9246-275F65C0E498";
  30. my $MSCODEPAGES = "$MSDATA/Windows Supported Code Page Data Files.zip";
  31. # Sort keys file
  32. my $SORTKEYS = "tr10/allkeys.txt";
  33. # Default char for undefined mappings
  34. my $DEF_CHAR = ord '?';
  35. # Last valid Unicode character
  36. my $MAX_CHAR = 0x10ffff;
  37. my @allfiles =
  38. (
  39. "CodpageFiles/037.txt",
  40. "CodpageFiles/437.txt",
  41. "CodpageFiles/500.txt",
  42. "CodpageFiles/708.txt",
  43. "CodpageFiles/720.txt",
  44. "CodpageFiles/737.txt",
  45. "CodpageFiles/775.txt",
  46. "CodpageFiles/850.txt",
  47. "CodpageFiles/852.txt",
  48. "CodpageFiles/855.txt",
  49. "CodpageFiles/857.txt",
  50. "CodpageFiles/860.txt",
  51. "CodpageFiles/861.txt",
  52. "CodpageFiles/862.txt",
  53. "CodpageFiles/863.txt",
  54. "CodpageFiles/864.txt",
  55. "CodpageFiles/865.txt",
  56. "CodpageFiles/866.txt",
  57. "CodpageFiles/869.txt",
  58. "CodpageFiles/874.txt",
  59. "CodpageFiles/875.txt",
  60. "CodpageFiles/932.txt",
  61. "CodpageFiles/936.txt",
  62. "CodpageFiles/949.txt",
  63. "CodpageFiles/950.txt",
  64. "CodpageFiles/1026.txt",
  65. "CodpageFiles/1250.txt",
  66. "CodpageFiles/1251.txt",
  67. "CodpageFiles/1252.txt",
  68. "CodpageFiles/1253.txt",
  69. "CodpageFiles/1254.txt",
  70. "CodpageFiles/1255.txt",
  71. "CodpageFiles/1256.txt",
  72. "CodpageFiles/1257.txt",
  73. "CodpageFiles/1258.txt",
  74. "CodpageFiles/1361.txt",
  75. "CodpageFiles/10000.txt",
  76. "CodpageFiles/10001.txt",
  77. "CodpageFiles/10002.txt",
  78. "CodpageFiles/10003.txt",
  79. "CodpageFiles/10004.txt",
  80. "CodpageFiles/10005.txt",
  81. "CodpageFiles/10006.txt",
  82. "CodpageFiles/10007.txt",
  83. "CodpageFiles/10008.txt",
  84. "CodpageFiles/10010.txt",
  85. "CodpageFiles/10017.txt",
  86. "CodpageFiles/10021.txt",
  87. "CodpageFiles/10029.txt",
  88. "CodpageFiles/10079.txt",
  89. "CodpageFiles/10081.txt",
  90. "CodpageFiles/10082.txt",
  91. "CodpageFiles/20127.txt",
  92. "CodpageFiles/20866.txt",
  93. "CodpageFiles/21866.txt",
  94. "CodpageFiles/28591.txt",
  95. "CodpageFiles/28592.txt",
  96. "CodpageFiles/28593.txt",
  97. "CodpageFiles/28594.txt",
  98. "CodpageFiles/28595.txt",
  99. "CodpageFiles/28596.txt",
  100. "CodpageFiles/28597.txt",
  101. "CodpageFiles/28598.txt",
  102. "CodpageFiles/28599.txt",
  103. "CodpageFiles/28603.txt",
  104. "CodpageFiles/28605.txt",
  105. );
  106. my %ctype =
  107. (
  108. # CT_CTYPE1
  109. "upper" => 0x0001,
  110. "lower" => 0x0002,
  111. "digit" => 0x0004,
  112. "space" => 0x0008,
  113. "punct" => 0x0010,
  114. "cntrl" => 0x0020,
  115. "blank" => 0x0040,
  116. "xdigit" => 0x0080,
  117. "alpha" => 0x0100 | 0x80000000,
  118. "defin" => 0x0200,
  119. # CT_CTYPE3 in high 16 bits
  120. "nonspacing" => 0x00010000,
  121. "diacritic" => 0x00020000,
  122. "vowelmark" => 0x00040000,
  123. "symbol" => 0x00080000,
  124. "katakana" => 0x00100000,
  125. "hiragana" => 0x00200000,
  126. "halfwidth" => 0x00400000,
  127. "fullwidth" => 0x00800000,
  128. "ideograph" => 0x01000000,
  129. "kashida" => 0x02000000,
  130. "lexical" => 0x04000000,
  131. "highsurrogate" => 0x08000000,
  132. "lowsurrogate" => 0x10000000,
  133. );
  134. my %bracket_types =
  135. (
  136. "o" => 0x0000,
  137. "c" => 0x0001,
  138. );
  139. my %indic_types =
  140. (
  141. "Other" => 0x0000,
  142. "Bindu" => 0x0001,
  143. "Visarga" => 0x0002,
  144. "Avagraha" => 0x0003,
  145. "Nukta" => 0x0004,
  146. "Virama" => 0x0005,
  147. "Vowel_Independent" => 0x0006,
  148. "Vowel_Dependent" => 0x0007,
  149. "Vowel" => 0x0008,
  150. "Consonant_Placeholder" => 0x0009,
  151. "Consonant" => 0x000a,
  152. "Consonant_Dead" => 0x000b,
  153. "Consonant_Succeeding_Repha" => 0x000c,
  154. "Consonant_Subjoined" => 0x000d,
  155. "Consonant_Medial" => 0x000e,
  156. "Consonant_Final" => 0x000f,
  157. "Consonant_Head_Letter" => 0x0010,
  158. "Modifying_Letter" => 0x0011,
  159. "Tone_Letter" => 0x0012,
  160. "Tone_Mark" => 0x0013,
  161. "Register_Shifter" => 0x0014,
  162. "Consonant_Preceding_Repha" => 0x0015,
  163. "Pure_Killer" => 0x0016,
  164. "Invisible_Stacker" => 0x0017,
  165. "Gemination_Mark" => 0x0018,
  166. "Cantillation_Mark" => 0x0019,
  167. "Non_Joiner" => 0x001a,
  168. "Joiner" => 0x001b,
  169. "Number_Joiner" => 0x001c,
  170. "Number" => 0x001d,
  171. "Brahmi_Joining_Number" => 0x001e,
  172. "Consonant_With_Stacker" => 0x001f,
  173. "Consonant_Prefixed" => 0x0020,
  174. "Syllable_Modifier" => 0x0021,
  175. "Consonant_Killer" => 0x0022,
  176. "Consonant_Initial_Postfixed" => 0x0023,
  177. );
  178. my %matra_types =
  179. (
  180. "Right" => 0x01,
  181. "Left" => 0x02,
  182. "Visual_Order_Left" => 0x03,
  183. "Left_And_Right" => 0x04,
  184. "Top" => 0x05,
  185. "Bottom" => 0x06,
  186. "Top_And_Bottom" => 0x07,
  187. "Top_And_Right" => 0x08,
  188. "Top_And_Left" => 0x09,
  189. "Top_And_Left_And_Right" => 0x0a,
  190. "Bottom_And_Right" => 0x0b,
  191. "Top_And_Bottom_And_Right" => 0x0c,
  192. "Overstruck" => 0x0d,
  193. "Invisible" => 0x0e,
  194. "Bottom_And_Left" => 0x0f,
  195. "Top_And_Bottom_And_Left" => 0x10,
  196. );
  197. my %break_types =
  198. (
  199. "BK" => 0x0001,
  200. "CR" => 0x0002,
  201. "LF" => 0x0003,
  202. "CM" => 0x0004,
  203. "SG" => 0x0005,
  204. "GL" => 0x0006,
  205. "CB" => 0x0007,
  206. "SP" => 0x0008,
  207. "ZW" => 0x0009,
  208. "NL" => 0x000a,
  209. "WJ" => 0x000b,
  210. "JL" => 0x000c,
  211. "JV" => 0x000d,
  212. "JT" => 0x000e,
  213. "H2" => 0x000f,
  214. "H3" => 0x0010,
  215. "XX" => 0x0011,
  216. "OP" => 0x0012,
  217. "CL" => 0x0013,
  218. "CP" => 0x0014,
  219. "QU" => 0x0015,
  220. "NS" => 0x0016,
  221. "EX" => 0x0017,
  222. "SY" => 0x0018,
  223. "IS" => 0x0019,
  224. "PR" => 0x001a,
  225. "PO" => 0x001b,
  226. "NU" => 0x001c,
  227. "AL" => 0x001d,
  228. "ID" => 0x001e,
  229. "IN" => 0x001f,
  230. "HY" => 0x0020,
  231. "BB" => 0x0021,
  232. "BA" => 0x0022,
  233. "SA" => 0x0023,
  234. "AI" => 0x0024,
  235. "B2" => 0x0025,
  236. "HL" => 0x0026,
  237. "CJ" => 0x0027,
  238. "RI" => 0x0028,
  239. "EB" => 0x0029,
  240. "EM" => 0x002a,
  241. "ZWJ" => 0x002b,
  242. );
  243. my %vertical_types =
  244. (
  245. "R" => 0x0000,
  246. "U" => 0x0001,
  247. "Tr" => 0x0002,
  248. "Tu" => 0x0003,
  249. );
  250. my %categories =
  251. (
  252. "Lu" => $ctype{"defin"}|$ctype{"alpha"}|$ctype{"upper"}, # Letter, Uppercase
  253. "Ll" => $ctype{"defin"}|$ctype{"alpha"}|$ctype{"lower"}, # Letter, Lowercase
  254. "Lt" => $ctype{"defin"}|$ctype{"alpha"}|$ctype{"upper"}|$ctype{"lower"}, # Letter, Titlecase
  255. "Mn" => $ctype{"defin"}|$ctype{"nonspacing"}, # Mark, Non-Spacing
  256. "Mc" => $ctype{"defin"}, # Mark, Spacing Combining
  257. "Me" => $ctype{"defin"}, # Mark, Enclosing
  258. "Nd" => $ctype{"defin"}|$ctype{"digit"}, # Number, Decimal Digit
  259. "Nl" => $ctype{"defin"}|$ctype{"alpha"}, # Number, Letter
  260. "No" => $ctype{"defin"}, # Number, Other
  261. "Zs" => $ctype{"defin"}|$ctype{"space"}, # Separator, Space
  262. "Zl" => $ctype{"defin"}|$ctype{"space"}, # Separator, Line
  263. "Zp" => $ctype{"defin"}|$ctype{"space"}, # Separator, Paragraph
  264. "Cc" => $ctype{"defin"}|$ctype{"cntrl"}, # Other, Control
  265. "Cf" => $ctype{"defin"}|$ctype{"cntrl"}, # Other, Format
  266. "Cs" => $ctype{"defin"}, # Other, Surrogate
  267. "Co" => $ctype{"defin"}, # Other, Private Use
  268. "Cn" => $ctype{"defin"}, # Other, Not Assigned
  269. "Lm" => $ctype{"defin"}|$ctype{"alpha"}, # Letter, Modifier
  270. "Lo" => $ctype{"defin"}|$ctype{"alpha"}, # Letter, Other
  271. "Pc" => $ctype{"defin"}|$ctype{"punct"}, # Punctuation, Connector
  272. "Pd" => $ctype{"defin"}|$ctype{"punct"}, # Punctuation, Dash
  273. "Ps" => $ctype{"defin"}|$ctype{"punct"}, # Punctuation, Open
  274. "Pe" => $ctype{"defin"}|$ctype{"punct"}, # Punctuation, Close
  275. "Pi" => $ctype{"defin"}|$ctype{"punct"}, # Punctuation, Initial quote
  276. "Pf" => $ctype{"defin"}|$ctype{"punct"}, # Punctuation, Final quote
  277. "Po" => $ctype{"defin"}|$ctype{"punct"}, # Punctuation, Other
  278. "Sm" => $ctype{"defin"}|$ctype{"symbol"}, # Symbol, Math
  279. "Sc" => $ctype{"defin"}|$ctype{"symbol"}, # Symbol, Currency
  280. "Sk" => $ctype{"defin"}|$ctype{"symbol"}, # Symbol, Modifier
  281. "So" => $ctype{"defin"}|$ctype{"symbol"} # Symbol, Other
  282. );
  283. # a few characters need additional categories that cannot be determined automatically
  284. my %special_categories =
  285. (
  286. "xdigit" => [ ord('0')..ord('9'),ord('A')..ord('F'),ord('a')..ord('f'),
  287. 0xff10..0xff19, 0xff21..0xff26, 0xff41..0xff46 ],
  288. "space" => [ 0x09..0x0d, 0x85 ],
  289. "blank" => [ 0x09, 0x20, 0xa0, 0x3000, 0xfeff ],
  290. "cntrl" => [ 0x070f, 0x200c, 0x200d,
  291. 0x200e, 0x200f, 0x202a, 0x202b, 0x202c, 0x202d, 0x202e,
  292. 0x206a, 0x206b, 0x206c, 0x206d, 0x206e, 0x206f, 0xfeff,
  293. 0xfff9, 0xfffa, 0xfffb ],
  294. "punct" => [ 0x24, 0x2b, 0x3c..0x3e, 0x5e, 0x60, 0x7c, 0x7e, 0xa2..0xbe,
  295. 0xd7, 0xf7 ],
  296. "digit" => [ 0xb2, 0xb3, 0xb9 ],
  297. "lower" => [ 0xaa, 0xba, 0x2071, 0x207f ],
  298. "nonspacing" => [ 0xc0..0xc5, 0xc7..0xcf, 0xd1..0xd6, 0xd8..0xdd, 0xe0..0xe5, 0xe7..0xef,
  299. 0xf1..0xf6, 0xf8..0xfd, 0xff, 0x6de, 0x1929..0x192b, 0x302e..0x302f ],
  300. "diacritic" => [ 0x5e, 0x60, 0xb7, 0xd8, 0xf8 ],
  301. "symbol" => [ 0x09..0x0d, 0x20..0x23, 0x25, 0x26, 0x28..0x2a, 0x2c, 0x2e..0x2f, 0x3a..0x40,
  302. 0x5b..0x60, 0x7b..0x7e, 0xa0..0xa9, 0xab..0xb1, 0xb4..0xb8, 0xbb, 0xbf,
  303. 0x02b9..0x02ba, 0x02c6..0x02cf ],
  304. "halfwidth" => [ 0x20..0x7e, 0xa2..0xa3, 0xa5..0xa6, 0xac, 0xaf, 0x20a9 ],
  305. "fullwidth" => [ 0x2018..0x2019, 0x201c..0x201d, 0x3000..0x3002, 0x300c..0x300d, 0x309b..0x309c,
  306. 0x30a1..0x30ab, 0x30ad, 0x30ad, 0x30af, 0x30b1, 0x30b3, 0x30b5, 0x30b7, 0x30b9,
  307. 0x30bb, 0x30bd, 0x30bf, 0x30c1, 0x30c3, 0x30c4, 0x30c6, 0x30c8, 0x30ca..0x30cf,
  308. 0x30d2, 0x30d5, 0x30d8, 0x30db, 0x30de..0x30ed, 0x30ef, 0x30f2..0x30f3, 0x30fb,
  309. 0x3131..0x3164 ],
  310. "ideograph" => [ 0x3006..0x3007 ],
  311. "lexical" => [ 0x22, 0x24, 0x27, 0x2d, 0x2f, 0x3d, 0x40, 0x5c, 0x5e..0x60, 0x7e,
  312. 0xa8, 0xaa, 0xad, 0xaf, 0xb4, 0xb8, 0xba,
  313. 0x02b0..0x02b8, 0x02bc, 0x02c7, 0x02ca..0x02cb, 0x02cf, 0x02d8..0x02dd, 0x02e0..0x02e3,
  314. 0x037a, 0x0384..0x0385, 0x0387, 0x0559..0x055a, 0x0640, 0x1fbd..0x1fc1,
  315. 0x1fcd..0x1fcf, 0x1fdd..0x1fdf, 0x1fed..0x1fef, 0x1ffd..0x1ffe, 0x2010..0x2015,
  316. 0x2032..0x2034, 0x2038, 0x2043..0x2044, 0x207b..0x207c, 0x207f, 0x208b..0x208c,
  317. 0x2212, 0x2215..0x2216, 0x2500, 0x2504..0x2505, 0x2508..0x2509, 0x254c..0x254d,
  318. 0x3003, 0x301c, 0x3030..0x3035, 0x309b..0x309e, 0x30fd..0x30fe, 0xfe31..0xfe32,
  319. 0xfe58, 0xfe63, 0xfe66, 0xfe68..0xfe69, 0xfe6b, 0xff04, 0xff07, 0xff0d, 0xff0f,
  320. 0xff1d, 0xff20, 0xff3c, 0xff3e, 0xff40, 0xff5e ],
  321. "kashida" => [ 0x0640 ],
  322. );
  323. my %directions =
  324. (
  325. "L" => 1, # Left-to-Right
  326. "R" => 2, # Right-to-Left
  327. "AL" => 12, # Right-to-Left Arabic
  328. "EN" => 3, # European Number
  329. "ES" => 4, # European Number Separator
  330. "ET" => 5, # European Number Terminator
  331. "AN" => 6, # Arabic Number
  332. "CS" => 7, # Common Number Separator
  333. "NSM" => 13, # Non-Spacing Mark
  334. "BN" => 14, # Boundary Neutral
  335. "B" => 8, # Paragraph Separator
  336. "S" => 9, # Segment Separator
  337. "WS" => 10, # Whitespace
  338. "ON" => 11, # Other Neutrals
  339. "LRE" => 15, # Left-to-Right Embedding
  340. "LRO" => 15, # Left-to-Right Override
  341. "RLE" => 15, # Right-to-Left Embedding
  342. "RLO" => 15, # Right-to-Left Override
  343. "PDF" => 15, # Pop Directional Format
  344. "LRI" => 15, # Left-to-Right Isolate
  345. "RLI" => 15, # Right-to-Left Isolate
  346. "FSI" => 15, # First Strong Isolate
  347. "PDI" => 15 # Pop Directional Isolate
  348. );
  349. my %c2_types =
  350. (
  351. "L" => 1, # C2_LEFTTORIGHT
  352. "R" => 2, # C2_RIGHTTOLEFT
  353. "AL" => 2, # C2_RIGHTTOLEFT
  354. "EN" => 3, # C2_EUROPENUMBER
  355. "ES" => 4, # C2_EUROPESEPARATOR
  356. "ET" => 5, # C2_EUROPETERMINATOR
  357. "AN" => 6, # C2_ARABICNUMBER
  358. "CS" => 7, # C2_COMMONSEPARATOR
  359. "NSM" => 11, # C2_OTHERNEUTRAL
  360. "BN" => 0, # C2_NOTAPPLICABLE
  361. "B" => 8, # C2_BLOCKSEPARATOR
  362. "S" => 9, # C2_SEGMENTSEPARATOR
  363. "WS" => 10, # C2_WHITESPACE
  364. "ON" => 11, # C2_OTHERNEUTRAL
  365. "LRE" => 11, # C2_OTHERNEUTRAL
  366. "LRO" => 11, # C2_OTHERNEUTRAL
  367. "RLE" => 11, # C2_OTHERNEUTRAL
  368. "RLO" => 11, # C2_OTHERNEUTRAL
  369. "PDF" => 11, # C2_OTHERNEUTRAL
  370. "LRI" => 11, # C2_OTHERNEUTRAL
  371. "RLI" => 11, # C2_OTHERNEUTRAL
  372. "FSI" => 11, # C2_OTHERNEUTRAL
  373. "PDI" => 11 # C2_OTHERNEUTRAL
  374. );
  375. my %bidi_types =
  376. (
  377. "ON" => 0, # Other Neutrals
  378. "L" => 1, # Left-to-Right
  379. "R" => 2, # Right-to-Left
  380. "AN" => 3, # Arabic Number
  381. "EN" => 4, # European Number
  382. "AL" => 5, # Right-to-Left Arabic
  383. "NSM" => 6, # Non-Spacing Mark
  384. "CS" => 7, # Common Number Separator
  385. "ES" => 8, # European Number Separator
  386. "ET" => 9, # European Number Terminator
  387. "BN" => 10, # Boundary Neutral
  388. "S" => 11, # Segment Separator
  389. "WS" => 12, # Whitespace
  390. "B" => 13, # Paragraph Separator
  391. "RLO" => 14, # Right-to-Left Override
  392. "RLE" => 15, # Right-to-Left Embedding
  393. "LRO" => 16, # Left-to-Right Override
  394. "LRE" => 17, # Left-to-Right Embedding
  395. "PDF" => 18, # Pop Directional Format
  396. "LRI" => 19, # Left-to-Right Isolate
  397. "RLI" => 20, # Right-to-Left Isolate
  398. "FSI" => 21, # First Strong Isolate
  399. "PDI" => 22 # Pop Directional Isolate
  400. );
  401. my %joining_types =
  402. (
  403. "U" => 0, # Non_Joining
  404. "L" => 1, # Left_Joining
  405. "R" => 2, # Right_Joining
  406. "D" => 3, # Dual_Joining
  407. "C" => 3, # Join_Causing
  408. "ALAPH" => 4, # Syriac ALAPH
  409. "DALATH RISH" => 5, # Syriac DALATH RISH group
  410. "T" => 6, # Transparent
  411. );
  412. my @cp2uni = ();
  413. my @glyph2uni = ();
  414. my @lead_bytes = ();
  415. my @uni2cp = ();
  416. my @tolower_table = ();
  417. my @toupper_table = ();
  418. my @digitmap_table = ();
  419. my @category_table = ();
  420. my @initial_joining_table = ();
  421. my @direction_table = ();
  422. my @decomp_table = ();
  423. my @combining_class_table = ();
  424. my @decomp_compat_table = ();
  425. my @comp_exclusions = ();
  426. my @idna_decomp_table = ();
  427. my @idna_disallowed = ();
  428. my %registry_keys;
  429. my $default_char;
  430. my $default_wchar;
  431. my %joining_forms =
  432. (
  433. "isolated" => [],
  434. "final" => [],
  435. "initial" => [],
  436. "medial" => []
  437. );
  438. sub to_utf16(@)
  439. {
  440. my @ret;
  441. foreach my $ch (@_)
  442. {
  443. if ($ch < 0x10000)
  444. {
  445. push @ret, $ch;
  446. }
  447. else
  448. {
  449. my $val = $ch - 0x10000;
  450. push @ret, 0xd800 | ($val >> 10), 0xdc00 | ($val & 0x3ff);
  451. }
  452. }
  453. return @ret;
  454. }
  455. ################################################################
  456. # fetch a unicode.org file and open it
  457. sub open_data_file($$)
  458. {
  459. my ($base, $name) = @_;
  460. my $cache = ($ENV{XDG_CACHE_HOME} || "$ENV{HOME}/.cache") . "/wine";
  461. (my $dir = "$cache/$name") =~ s/\/[^\/]+$//;
  462. my $suffix = ($base =~ /\/\Q$UNIVERSION\E/) ? "-$UNIVERSION" : "";
  463. local *FILE;
  464. if ($base =~ /.*\/([^\/]+)\.zip$/)
  465. {
  466. my $zip = "$1$suffix.zip";
  467. unless (-f "$cache/$zip")
  468. {
  469. system "mkdir", "-p", $cache;
  470. print "Fetching $base...\n";
  471. !system "wget", "-q", "-O", "$cache/$zip", $base or die "cannot fetch $base";
  472. }
  473. open FILE, "-|", "unzip", "-p", "$cache/$zip", $name or die "cannot extract $name from $zip";
  474. }
  475. else
  476. {
  477. (my $dest = "$cache/$name") =~ s/(.*)(\.[^\/.]+)$/$1$suffix$2/;
  478. unless (-f $dest)
  479. {
  480. system "mkdir", "-p", $dir;
  481. print "Fetching $base/$name...\n";
  482. !system "wget", "-q", "-O", $dest, "$base/$name" or die "cannot fetch $base/$name";
  483. }
  484. open FILE, "<$dest" or die "cannot open $dest";
  485. }
  486. return *FILE;
  487. }
  488. ################################################################
  489. # recursively get the decomposition for a character
  490. sub get_decomposition($$);
  491. sub get_decomposition($$)
  492. {
  493. my ($char, $table) = @_;
  494. my @ret;
  495. return $char unless defined $table->[$char];
  496. foreach my $ch (@{$table->[$char]})
  497. {
  498. push @ret, get_decomposition( $ch, $table );
  499. }
  500. return @ret;
  501. }
  502. ################################################################
  503. # get the composition that results in a given character
  504. sub get_composition($$)
  505. {
  506. my ($ch, $compat) = @_;
  507. return () unless defined $decomp_table[$ch]; # no decomposition
  508. my @ret = @{$decomp_table[$ch]};
  509. return () if @ret < 2; # singleton decomposition
  510. return () if $comp_exclusions[$ch]; # composition exclusion
  511. return () if $combining_class_table[$ch]; # non-starter
  512. return () if $combining_class_table[$ret[0]]; # first char is non-starter
  513. return () if $compat == 1 && !defined $decomp_table[$ret[0]] &&
  514. defined $decomp_compat_table[$ret[0]]; # first char has compat decomposition
  515. return () if $compat == 2 && !defined $decomp_table[$ret[0]] &&
  516. defined $idna_decomp_table[$ret[0]]; # first char has IDNA decomposition
  517. return () if $compat == 2 && defined $idna_decomp_table[$ret[0]] &&
  518. defined $idna_decomp_table[$idna_decomp_table[$ret[0]]->[0]]; # first char's decomposition has IDNA decomposition
  519. return () if $compat == 2 && defined $idna_decomp_table[$ret[1]]; # second char has IDNA decomposition
  520. return @ret;
  521. }
  522. ################################################################
  523. # recursively build decompositions
  524. sub build_decompositions(@)
  525. {
  526. my @src = @_;
  527. my @dst;
  528. for (my $i = 0; $i < @src; $i++)
  529. {
  530. next unless defined $src[$i];
  531. my @decomp = to_utf16( get_decomposition( $i, \@src ));
  532. $dst[$i] = \@decomp;
  533. }
  534. return @dst;
  535. }
  536. ################################################################
  537. # compose Hangul sequences
  538. sub compose_hangul(@)
  539. {
  540. my $SBASE = 0xac00;
  541. my $LBASE = 0x1100;
  542. my $VBASE = 0x1161;
  543. my $TBASE = 0x11a7;
  544. my $LCOUNT = 19;
  545. my $VCOUNT = 21;
  546. my $TCOUNT = 28;
  547. my $NCOUNT = $VCOUNT * $TCOUNT;
  548. my $SCOUNT = $LCOUNT * $NCOUNT;
  549. my @seq = @_;
  550. my @ret;
  551. my $i;
  552. for ($i = 0; $i < @seq; $i++)
  553. {
  554. my $ch = $seq[$i];
  555. if ($ch >= $LBASE && $ch < $LBASE + $LCOUNT && $i < @seq - 1 &&
  556. $seq[$i+1] >= $VBASE && $seq[$i+1] < $VBASE + $VCOUNT)
  557. {
  558. $ch = $SBASE + (($seq[$i] - $LBASE) * $VCOUNT + ($seq[$i+1] - $VBASE)) * $TCOUNT;
  559. $i++;
  560. }
  561. if ($ch >= $SBASE && $ch < $SBASE + $SCOUNT && !(($ch - $SBASE) % $TCOUNT) && $i < @seq - 1 &&
  562. $seq[$i+1] > $TBASE && $seq[$i+1] < $TBASE + $TCOUNT)
  563. {
  564. $ch += $seq[$i+1] - $TBASE;
  565. $i++;
  566. }
  567. push @ret, $ch;
  568. }
  569. return @ret;
  570. }
  571. ################################################################
  572. # remove linguistic-only mappings from the case table
  573. sub remove_linguistic_mappings($$)
  574. {
  575. my ($upper, $lower) = @_;
  576. # remove case mappings that don't round-trip
  577. for (my $i = 0; $i < @{$upper}; $i++)
  578. {
  579. next unless defined ${$upper}[$i];
  580. my $ch = ${$upper}[$i];
  581. ${$upper}[$i] = undef unless defined ${$lower}[$ch] && ${$lower}[$ch] == $i;
  582. }
  583. for (my $i = 0; $i < @{$lower}; $i++)
  584. {
  585. next unless defined ${$lower}[$i];
  586. my $ch = ${$lower}[$i];
  587. ${$lower}[$i] = undef unless defined ${$upper}[$ch] && ${$upper}[$ch] == $i;
  588. }
  589. }
  590. ################################################################
  591. # read in the Unicode database files
  592. sub load_data()
  593. {
  594. my $start;
  595. # now build mappings from the decomposition field of the Unicode database
  596. my $UNICODE_DATA = open_data_file( $UNIDATA, "UnicodeData.txt" );
  597. while (<$UNICODE_DATA>)
  598. {
  599. # Decode the fields ...
  600. my ($code, $name, $cat, $comb, $bidi,
  601. $decomp, $dec, $dig, $num, $mirror,
  602. $oldname, $comment, $upper, $lower, $title) = split /;/;
  603. my $src = hex $code;
  604. die "unknown category $cat" unless defined $categories{$cat};
  605. die "unknown directionality $bidi" unless defined $directions{$bidi};
  606. $category_table[$src] = $categories{$cat};
  607. $direction_table[$src] = $bidi;
  608. if ($cat eq "Mn" || $cat eq "Me" || $cat eq "Cf")
  609. {
  610. $initial_joining_table[$src] = $joining_types{"T"};
  611. }
  612. else
  613. {
  614. $initial_joining_table[$src] = $joining_types{"U"};
  615. }
  616. if ($lower ne "")
  617. {
  618. $tolower_table[$src] = hex $lower;
  619. }
  620. if ($upper ne "")
  621. {
  622. $toupper_table[$src] = hex $upper;
  623. }
  624. if ($dec ne "")
  625. {
  626. $category_table[$src] |= $ctype{"digit"};
  627. }
  628. if ($dig ne "")
  629. {
  630. $digitmap_table[$src] = ord $dig;
  631. }
  632. $combining_class_table[$src] = ($cat ne "Co") ? $comb : 0x100; # Private Use
  633. $category_table[$src] |= $ctype{"nonspacing"} if $bidi eq "NSM";
  634. $category_table[$src] |= $ctype{"diacritic"} if $name =~ /^(COMBINING)|(MODIFIER LETTER)\W/;
  635. $category_table[$src] |= $ctype{"vowelmark"} if $name =~ /\sVOWEL/ || $oldname =~ /\sVOWEL/;
  636. $category_table[$src] |= $ctype{"halfwidth"} if $name =~ /^HALFWIDTH\s/;
  637. $category_table[$src] |= $ctype{"fullwidth"} if $name =~ /^FULLWIDTH\s/;
  638. $category_table[$src] |= $ctype{"hiragana"} if $name =~ /(HIRAGANA)|(\WKANA\W)/;
  639. $category_table[$src] |= $ctype{"katakana"} if $name =~ /(KATAKANA)|(\WKANA\W)/;
  640. $category_table[$src] |= $ctype{"ideograph"} if $name =~ /^<CJK Ideograph/;
  641. $category_table[$src] |= $ctype{"ideograph"} if $name =~ /^CJK COMPATIBILITY IDEOGRAPH/;
  642. $category_table[$src] |= $ctype{"ideograph"} if $name =~ /^HANGZHOU/;
  643. $category_table[$src] |= $ctype{"highsurrogate"} if $name =~ /High Surrogate/;
  644. $category_table[$src] |= $ctype{"lowsurrogate"} if $name =~ /Low Surrogate/;
  645. # copy the category and direction for everything between First/Last pairs
  646. if ($name =~ /, First>/) { $start = $src; }
  647. if ($name =~ /, Last>/)
  648. {
  649. while ($start < $src)
  650. {
  651. $category_table[$start] = $category_table[$src];
  652. $direction_table[$start] = $direction_table[$src];
  653. $combining_class_table[$start] = $combining_class_table[$src];
  654. $start++;
  655. }
  656. }
  657. next if $decomp eq ""; # no decomposition, skip it
  658. if ($decomp =~ /^<([a-zA-Z]+)>\s+([0-9a-fA-F]+)/)
  659. {
  660. my @seq = map { hex $_; } (split /\s+/, (split /\s+/, $decomp, 2)[1]);
  661. $decomp_compat_table[$src] = \@seq;
  662. }
  663. if ($decomp =~ /^<([a-zA-Z]+)>\s+([0-9a-fA-F]+)$/)
  664. {
  665. # decomposition of the form "<foo> 1234" -> use char if type is known
  666. if ($1 eq "isolated" || $1 eq "final" || $1 eq "initial" || $1 eq "medial")
  667. {
  668. ${joining_forms{$1}}[hex $2] = $src;
  669. }
  670. }
  671. elsif ($decomp =~ /^<compat>\s+0020\s+([0-9a-fA-F]+)/)
  672. {
  673. # decomposition "<compat> 0020 1234" -> combining accent
  674. }
  675. elsif ($decomp =~ /^([0-9a-fA-F]+)/)
  676. {
  677. # store decomposition
  678. if ($decomp =~ /^([0-9a-fA-F]+)\s+([0-9a-fA-F]+)$/)
  679. {
  680. $decomp_table[$src] = $decomp_compat_table[$src] = [ hex $1, hex $2 ];
  681. }
  682. elsif ($decomp =~ /^([0-9a-fA-F]+)$/)
  683. {
  684. # Single char decomposition
  685. $decomp_table[$src] = $decomp_compat_table[$src] = [ hex $1 ];
  686. }
  687. }
  688. }
  689. close $UNICODE_DATA;
  690. # patch the category of some special characters
  691. for (my $i = 0; $i < @decomp_table; $i++)
  692. {
  693. next unless defined $decomp_table[$i];
  694. $category_table[$i] |= $category_table[$decomp_table[$i]->[0]];
  695. }
  696. foreach my $cat (keys %special_categories)
  697. {
  698. my $flag = $ctype{$cat};
  699. foreach my $i (@{$special_categories{$cat}}) { $category_table[$i] |= $flag; }
  700. }
  701. for (my $i = 0; $i < @decomp_compat_table; $i++)
  702. {
  703. next unless defined $decomp_compat_table[$i];
  704. next unless @{$decomp_compat_table[$i]} == 2;
  705. $category_table[$i] |= $category_table[$decomp_compat_table[$i]->[1]] & $ctype{"diacritic"};
  706. }
  707. # load the composition exclusions
  708. my $EXCL = open_data_file( $UNIDATA, "CompositionExclusions.txt" );
  709. while (<$EXCL>)
  710. {
  711. s/\#.*//; # remove comments
  712. if (/^([0-9a-fA-F]+)\.\.([0-9a-fA-F]+)\s*$/)
  713. {
  714. foreach my $i (hex $1 .. hex $2) { $comp_exclusions[$i] = 1; }
  715. }
  716. elsif (/^([0-9a-fA-F]+)\s*$/)
  717. {
  718. $comp_exclusions[hex $1] = 1;
  719. }
  720. }
  721. close $EXCL;
  722. # load the IDNA mappings
  723. @idna_decomp_table = @decomp_compat_table;
  724. my $IDNA = open_data_file( $IDNADATA, "IdnaMappingTable.txt" );
  725. while (<$IDNA>)
  726. {
  727. s/\#.*//; # remove comments
  728. next if /^\s*$/;
  729. my ($char, $type, $mapping) = split /;/;
  730. my ($ch1, $ch2);
  731. if ($char =~ /([0-9a-fA-F]+)\.\.([0-9a-fA-F]+)/)
  732. {
  733. $ch1 = hex $1;
  734. $ch2 = hex $2;
  735. }
  736. elsif ($char =~ /([0-9a-fA-F]+)/)
  737. {
  738. $ch1 = $ch2 = hex $1;
  739. }
  740. if ($type =~ /mapped/ || $type =~ /deviation/)
  741. {
  742. $mapping =~ s/^\s*(([0-9a-fA-F]+\s+)+)\s*$/$1/;
  743. my @seq = map { hex $_; } split /\s+/, $mapping;
  744. foreach my $i ($ch1 .. $ch2) { $idna_decomp_table[$i] = @seq ? \@seq : [ 0 ]; }
  745. }
  746. elsif ($type =~ /valid/)
  747. {
  748. }
  749. elsif ($type =~ /ignored/)
  750. {
  751. foreach my $i ($ch1 .. $ch2) { $idna_decomp_table[$i] = [ 0 ]; }
  752. }
  753. elsif ($type =~ /disallowed/)
  754. {
  755. foreach my $i ($ch1 .. $ch2)
  756. {
  757. $idna_decomp_table[$i] = undef;
  758. $idna_disallowed[$i] = 1;
  759. }
  760. }
  761. }
  762. close $IDNA;
  763. }
  764. ################################################################
  765. # add a new registry key
  766. sub add_registry_key($$)
  767. {
  768. my ($key, $defval) = @_;
  769. $registry_keys{$key} = [ $defval ] unless defined $registry_keys{$key};
  770. }
  771. ################################################################
  772. # add a new registry value
  773. sub add_registry_value($$$)
  774. {
  775. my ($key, $name, $value) = @_;
  776. add_registry_key( $key, undef );
  777. push @{$registry_keys{$key}}, "'$name' = s '$value'";
  778. }
  779. ################################################################
  780. # define a new lead byte
  781. sub add_lead_byte($)
  782. {
  783. my $ch = shift;
  784. return if defined $cp2uni[$ch];
  785. push @lead_bytes, $ch;
  786. $cp2uni[$ch] = 0;
  787. }
  788. ################################################################
  789. # define a new char mapping
  790. sub add_mapping($$)
  791. {
  792. my ($cp, $uni) = @_;
  793. $cp2uni[$cp] = $uni unless defined($cp2uni[$cp]);
  794. $uni2cp[$uni] = $cp unless defined($uni2cp[$uni]);
  795. if ($cp > 0xff) { add_lead_byte( $cp >> 8 ); }
  796. }
  797. ################################################################
  798. # get a mapping including glyph chars for MB_USEGLYPHCHARS
  799. sub get_glyphs_mapping(@)
  800. {
  801. my @table = @_;
  802. for (my $i = 0; $i < @glyph2uni; $i++)
  803. {
  804. $table[$i] = $glyph2uni[$i] if defined $glyph2uni[$i];
  805. }
  806. return @table;
  807. }
  808. ################################################################
  809. # build EUC-JP table from the JIS 0208/0212 files
  810. sub dump_eucjp_codepage()
  811. {
  812. @cp2uni = ();
  813. @glyph2uni = ();
  814. @lead_bytes = ();
  815. @uni2cp = ();
  816. $default_char = $DEF_CHAR;
  817. $default_wchar = 0x30fb;
  818. # ASCII chars
  819. foreach my $i (0x00 .. 0x7f) { add_mapping( $i, $i ); }
  820. # lead bytes
  821. foreach my $i (0x8e, 0xa1 .. 0xfe) { add_lead_byte($i); }
  822. # JIS X 0201 right plane
  823. foreach my $i (0xa1 .. 0xdf) { add_mapping( 0x8e00 + $i, 0xfec0 + $i ); }
  824. # undefined chars
  825. foreach my $i (0x80 .. 0x8d, 0x8f .. 0x9f) { $cp2uni[$i] = $i; }
  826. $cp2uni[0xa0] = 0xf8f0;
  827. $cp2uni[0xff] = 0xf8f3;
  828. # Fix backslash conversion
  829. add_mapping( 0xa1c0, 0xff3c );
  830. # Add private mappings for rows undefined in JIS 0208/0212
  831. my $private = 0xe000;
  832. foreach my $hi (0xf5 .. 0xfe)
  833. {
  834. foreach my $lo (0xa1 .. 0xfe)
  835. {
  836. add_mapping( ($hi << 8) + $lo, $private++ );
  837. }
  838. }
  839. foreach my $hi (0xf5 .. 0xfe)
  840. {
  841. foreach my $lo (0x21 .. 0x7e)
  842. {
  843. add_mapping( ($hi << 8) + $lo, $private++ );
  844. }
  845. }
  846. my $INPUT = open_data_file( $JISDATA, "JIS0208.TXT" );
  847. while (<$INPUT>)
  848. {
  849. next if /^\#/; # skip comments
  850. next if /^$/; # skip empty lines
  851. next if /\x1a/; # skip ^Z
  852. if (/^0x[0-9a-fA-F]+\s+0x([0-9a-fA-F]+)\s+0x([0-9a-fA-F]+)\s+(\#.*)?/)
  853. {
  854. add_mapping( 0x8080 + hex $1, hex $2 );
  855. next;
  856. }
  857. die "Unrecognized line $_\n";
  858. }
  859. close $INPUT;
  860. $INPUT = open_data_file( $JISDATA, "JIS0212.TXT" );
  861. while (<$INPUT>)
  862. {
  863. next if /^\#/; # skip comments
  864. next if /^$/; # skip empty lines
  865. next if /\x1a/; # skip ^Z
  866. if (/^0x([0-9a-fA-F]+)\s+0x([0-9a-fA-F]+)\s+(\#.*)?/)
  867. {
  868. add_mapping( 0x8000 + hex $1, hex $2 );
  869. next;
  870. }
  871. die "Unrecognized line $_\n";
  872. }
  873. close $INPUT;
  874. output_codepage_file( 20932 );
  875. }
  876. ################################################################
  877. # build Korean Wansung table from the KSX1001 file
  878. sub dump_krwansung_codepage(@)
  879. {
  880. my @cp949 = @_;
  881. @cp2uni = ();
  882. @glyph2uni = ();
  883. @lead_bytes = ();
  884. @uni2cp = ();
  885. $default_char = 0x3f;
  886. $default_wchar = 0x003f;
  887. # ASCII and undefined chars
  888. foreach my $i (0x00 .. 0x9f) { add_mapping( $i, $i ); }
  889. add_mapping( 0xa0, 0xf8e6 );
  890. add_mapping( 0xad, 0xf8e7 );
  891. add_mapping( 0xae, 0xf8e8 );
  892. add_mapping( 0xaf, 0xf8e9 );
  893. add_mapping( 0xfe, 0xf8ea );
  894. add_mapping( 0xff, 0xf8eb );
  895. my $INPUT = open_data_file( $KSCDATA, "KSX1001.TXT" );
  896. while (<$INPUT>)
  897. {
  898. next if /^\#/; # skip comments
  899. next if /^$/; # skip empty lines
  900. next if /\x1a/; # skip ^Z
  901. if (/^0x([0-9a-fA-F]+)\s+0x([0-9a-fA-F]+)\s+(\#.*)?/)
  902. {
  903. add_mapping( 0x8080 + hex $1, hex $2 );
  904. next;
  905. }
  906. die "Unrecognized line $_\n";
  907. }
  908. close $INPUT;
  909. # get some extra mappings from cp 949
  910. my @defined_lb;
  911. map { $defined_lb[$_] = 1; } @lead_bytes;
  912. foreach my $i (0x0000 .. 0xffff)
  913. {
  914. next if ($i >= 0x1100 && $i <= 0x11ff); # range not used in 20949
  915. next unless defined $cp949[$i];
  916. if ($cp949[$i] >= 0xff)
  917. {
  918. # only add chars for lead bytes that exist in 20949
  919. my $hi = $cp949[$i] >> 8;
  920. my $lo = $cp949[$i] & 0xff;
  921. next unless $defined_lb[$hi];
  922. next unless $lo >= 0xa1 && $lo <= 0xfe;
  923. }
  924. add_mapping( $cp949[$i], $i );
  925. }
  926. output_codepage_file( 20949 );
  927. }
  928. ################################################################
  929. # build the sort keys table
  930. sub dump_sortkeys($)
  931. {
  932. my $filename = shift;
  933. my @sortkeys = ();
  934. my $INPUT = open_data_file( $REPORTS, $SORTKEYS );
  935. while (<$INPUT>)
  936. {
  937. next if /^\#/; # skip comments
  938. next if /^$/; # skip empty lines
  939. next if /\x1a/; # skip ^Z
  940. next if /^\@version/; # skip @version header
  941. if (/^([0-9a-fA-F]+)\s+;\s+\[([*.])([0-9a-fA-F]{4})\.([0-9a-fA-F]{4})\.([0-9a-fA-F]{4})\.([0-9a-fA-F]+)\]/)
  942. {
  943. my ($uni,$variable) = (hex $1, $2);
  944. next if $uni > 65535;
  945. $sortkeys[$uni] = [ $uni, hex $3, hex $4, hex $5, hex $6 ];
  946. next;
  947. }
  948. if (/^([0-9a-fA-F]+\s+)+;\s+\[[*.]([0-9a-fA-F]{4})\.([0-9a-fA-F]{4})\.([0-9a-fA-F]{4})\.([0-9a-fA-F]+)\]/)
  949. {
  950. # multiple character sequence, ignored for now
  951. next;
  952. }
  953. die "$SORTKEYS: Unrecognized line $_\n";
  954. }
  955. close $INPUT;
  956. # compress the keys to 32 bit:
  957. # key 1 to 16 bits, key 2 to 8 bits, key 3 to 4 bits, key 4 to 1 bit
  958. @sortkeys = sort { ${$a}[1] <=> ${$b}[1] or
  959. ${$a}[2] <=> ${$b}[2] or
  960. ${$a}[3] <=> ${$b}[3] or
  961. ${$a}[4] <=> ${$b}[4] or
  962. $a cmp $b; } @sortkeys;
  963. my ($n2, $n3) = (1, 1);
  964. my @keys = (-1, -1, -1, -1, -1 );
  965. my @flatkeys = ();
  966. for (my $i = 0; $i < @sortkeys; $i++)
  967. {
  968. next unless defined $sortkeys[$i];
  969. my @current = @{$sortkeys[$i]};
  970. if ($current[1] == $keys[1])
  971. {
  972. if ($current[2] == $keys[2])
  973. {
  974. if ($current[3] == $keys[3])
  975. {
  976. # nothing
  977. }
  978. else
  979. {
  980. $keys[3] = $current[3];
  981. $n3++;
  982. die if ($n3 >= 16);
  983. }
  984. }
  985. else
  986. {
  987. $keys[2] = $current[2];
  988. $keys[3] = $current[3];
  989. $n2++;
  990. $n3 = 1;
  991. die if ($n2 >= 256);
  992. }
  993. }
  994. else
  995. {
  996. $keys[1] = $current[1];
  997. $keys[2] = $current[2];
  998. $keys[3] = $current[3];
  999. $n2 = 1;
  1000. $n3 = 1;
  1001. }
  1002. if ($current[2]) { $current[2] = $n2; }
  1003. if ($current[3]) { $current[3] = $n3; }
  1004. if ($current[4]) { $current[4] = 1; }
  1005. $flatkeys[$current[0]] = ($current[1] << 16) | ($current[2] << 8) | ($current[3] << 4) | $current[4];
  1006. }
  1007. open OUTPUT,">$filename.new" or die "Cannot create $filename";
  1008. printf "Building $filename\n";
  1009. printf OUTPUT "/* Unicode collation element table */\n";
  1010. printf OUTPUT "/* generated from %s */\n", "$REPORTS/$SORTKEYS";
  1011. printf OUTPUT "/* DO NOT EDIT!! */\n\n";
  1012. print OUTPUT "#include \"windef.h\"\n\n";
  1013. dump_two_level_mapping( "collation_table", 0xffffffff, 32, @flatkeys );
  1014. close OUTPUT;
  1015. save_file($filename);
  1016. }
  1017. ################################################################
  1018. # dump an array of integers
  1019. sub dump_array($$@)
  1020. {
  1021. my ($bit_width, $default, @array) = @_;
  1022. my $format = sprintf "0x%%0%ux", $bit_width / 4;
  1023. my $i;
  1024. my $ret = " ";
  1025. for ($i = 0; $i < $#array; $i++)
  1026. {
  1027. $ret .= sprintf($format, defined $array[$i] ? $array[$i] : $default);
  1028. $ret .= (($i % 8) != 7) ? ", " : ",\n ";
  1029. }
  1030. $ret .= sprintf($format, defined $array[$i] ? $array[$i] : $default);
  1031. return $ret;
  1032. }
  1033. ################################################################
  1034. # dump an SBCS mapping table in binary format
  1035. sub dump_binary_sbcs_table($)
  1036. {
  1037. my $codepage = shift;
  1038. my @header = ( 13, $codepage, 1, $default_char, $default_wchar, $cp2uni[$default_char], $uni2cp[$default_wchar] );
  1039. my $wc_offset = 256 + 3 + (@glyph2uni ? 256 : 0);
  1040. print OUTPUT pack "S<*", @header;
  1041. print OUTPUT pack "C12", (0) x 12;
  1042. print OUTPUT pack "S<*", $wc_offset, map { $_ || 0; } @cp2uni[0 .. 255];
  1043. if (@glyph2uni)
  1044. {
  1045. print OUTPUT pack "S<*", 256, get_glyphs_mapping(@cp2uni[0 .. 255]);
  1046. }
  1047. else
  1048. {
  1049. print OUTPUT pack "S<*", 0;
  1050. }
  1051. print OUTPUT pack "S<*", 0, 0;
  1052. print OUTPUT pack "C*", map { defined $_ ? $_ : $default_char; } @uni2cp[0 .. 65535];
  1053. }
  1054. ################################################################
  1055. # dump a DBCS mapping table in binary format
  1056. sub dump_binary_dbcs_table($)
  1057. {
  1058. my $codepage = shift;
  1059. my @lb_ranges = get_lb_ranges();
  1060. my @header = ( 13, $codepage, 2, $default_char, $default_wchar, $cp2uni[$default_char], $uni2cp[$default_wchar] );
  1061. my @offsets = (0) x 256;
  1062. my $pos = 0;
  1063. foreach my $i (@lead_bytes)
  1064. {
  1065. $offsets[$i] = ($pos += 256);
  1066. $cp2uni[$i] = 0;
  1067. }
  1068. my $wc_offset = 256 + 3 + 256 * (1 + scalar @lead_bytes);
  1069. print OUTPUT pack "S<*", @header;
  1070. print OUTPUT pack "C12", @lb_ranges, 0 x 12;
  1071. print OUTPUT pack "S<*", $wc_offset, map { $_ || 0; } @cp2uni[0 .. 255];
  1072. print OUTPUT pack "S<*", 0, scalar @lb_ranges / 2, @offsets;
  1073. foreach my $i (@lead_bytes)
  1074. {
  1075. my $base = $i << 8;
  1076. print OUTPUT pack "S<*", map { defined $_ ? $_ : $default_wchar; } @cp2uni[$base .. $base + 255];
  1077. }
  1078. print OUTPUT pack "S<", 4;
  1079. print OUTPUT pack "S<*", map { defined $_ ? $_ : $default_char; } @uni2cp[0 .. 65535];
  1080. }
  1081. ################################################################
  1082. # get the list of defined lead byte ranges
  1083. sub get_lb_ranges()
  1084. {
  1085. my @list = ();
  1086. my @ranges = ();
  1087. foreach my $i (@lead_bytes) { $list[$i] = 1; }
  1088. my $on = 0;
  1089. for (my $i = 0; $i < 256; $i++)
  1090. {
  1091. if ($on)
  1092. {
  1093. if (!defined $list[$i]) { push @ranges, $i-1; $on = 0; }
  1094. }
  1095. else
  1096. {
  1097. if ($list[$i]) { push @ranges, $i; $on = 1; }
  1098. }
  1099. }
  1100. if ($on) { push @ranges, 0xff; }
  1101. return @ranges;
  1102. }
  1103. ################################################################
  1104. # dump the Indic Syllabic Category table
  1105. sub dump_indic($)
  1106. {
  1107. my $filename = shift;
  1108. my @indic_table;
  1109. my $INPUT = open_data_file( $UNIDATA, "IndicSyllabicCategory.txt" );
  1110. while (<$INPUT>)
  1111. {
  1112. next if /^\#/; # skip comments
  1113. next if /^\s*$/; # skip empty lines
  1114. next if /\x1a/; # skip ^Z
  1115. if (/^\s*([0-9a-fA-F]+)\s*;\s*([a-zA-Z_]+)\s*#/)
  1116. {
  1117. my $type = $2;
  1118. die "unknown indic $type" unless defined $indic_types{$type};
  1119. if (hex $1 < 65536)
  1120. {
  1121. $indic_table[hex $1] = $indic_types{$type};
  1122. }
  1123. next;
  1124. }
  1125. elsif (/^\s*([0-9a-fA-F]+)\.\.\s*([0-9a-fA-F]+)\s*;\s*([A-Za-z_]+)\s*#/)
  1126. {
  1127. my $type = $3;
  1128. die "unknown indic $type" unless defined $indic_types{$type};
  1129. if (hex $1 < 65536 and hex $2 < 65536)
  1130. {
  1131. foreach my $i (hex $1 .. hex $2)
  1132. {
  1133. $indic_table[$i] = $indic_types{$type};
  1134. }
  1135. }
  1136. next;
  1137. }
  1138. die "malformed line $_";
  1139. }
  1140. close $INPUT;
  1141. $INPUT = open_data_file( $UNIDATA, "IndicPositionalCategory.txt" );
  1142. while (<$INPUT>)
  1143. {
  1144. next if /^\#/; # skip comments
  1145. next if /^\s*$/; # skip empty lines
  1146. next if /\x1a/; # skip ^Z
  1147. if (/^\s*([0-9a-fA-F]+)\s*;\s*([a-zA-Z_]+)\s*#/)
  1148. {
  1149. my $type = $2;
  1150. die "unknown matra $type" unless defined $matra_types{$type};
  1151. $indic_table[hex $1] |= $matra_types{$type} << 8;
  1152. next;
  1153. }
  1154. elsif (/^\s*([0-9a-fA-F]+)\.\.\s*([0-9a-fA-F]+)\s*;\s*([A-Za-z_]+)\s*#/)
  1155. {
  1156. my $type = $3;
  1157. die "unknown matra $type" unless defined $matra_types{$type};
  1158. foreach my $i (hex $1 .. hex $2)
  1159. {
  1160. $indic_table[$i] |= $matra_types{$type} << 8;
  1161. }
  1162. next;
  1163. }
  1164. die "malformed line $_";
  1165. }
  1166. close $INPUT;
  1167. open OUTPUT,">$filename.new" or die "Cannot create $filename";
  1168. print "Building $filename\n";
  1169. print OUTPUT "/* Unicode Indic Syllabic Category */\n";
  1170. print OUTPUT "/* generated from $UNIDATA:IndicSyllabicCategory.txt */\n";
  1171. print OUTPUT "/* and from $UNIDATA:IndicPositionalCategory.txt */\n";
  1172. print OUTPUT "/* DO NOT EDIT!! */\n\n";
  1173. print OUTPUT "#include \"windef.h\"\n\n";
  1174. dump_two_level_mapping( "indic_syllabic_table", $indic_types{'Other'}, 16, @indic_table );
  1175. close OUTPUT;
  1176. save_file($filename);
  1177. }
  1178. ################################################################
  1179. # dump the Line Break Properties table
  1180. sub dump_linebreak($)
  1181. {
  1182. my $filename = shift;
  1183. my @break_table;
  1184. my $INPUT = open_data_file( $UNIDATA, "LineBreak.txt" );
  1185. while (<$INPUT>)
  1186. {
  1187. next if /^\#/; # skip comments
  1188. next if /^\s*$/; # skip empty lines
  1189. next if /\x1a/; # skip ^Z
  1190. if (/^\s*([0-9a-fA-F]+)\s*;\s*([0-9A-Z][0-9A-Z][0-9A-Z])+\s*/)
  1191. {
  1192. my $type = $2;
  1193. die "unknown breaktype $type" unless defined $break_types{$type};
  1194. $break_table[hex $1] = $break_types{$type};
  1195. next;
  1196. }
  1197. elsif (/^\s*([0-9a-fA-F]+)\.\.\s*([0-9a-fA-F]+)\s*;\s*([0-9A-Z][0-9A-Z][0-9A-Z])+\s*/)
  1198. {
  1199. my $type = $3;
  1200. die "unknown breaktype $type" unless defined $break_types{$type};
  1201. foreach my $i (hex $1 .. hex $2)
  1202. {
  1203. $break_table[$i] = $break_types{$type};
  1204. }
  1205. next;
  1206. }
  1207. elsif (/^\s*([0-9a-fA-F]+)\s*;\s*([0-9A-Z][0-9A-Z])+\s*/)
  1208. {
  1209. my $type = $2;
  1210. die "unknown breaktype $type" unless defined $break_types{$type};
  1211. $break_table[hex $1] = $break_types{$type};
  1212. next;
  1213. }
  1214. elsif (/^\s*([0-9a-fA-F]+)\.\.\s*([0-9a-fA-F]+)\s*;\s*([0-9A-Z][0-9A-Z])+\s*/)
  1215. {
  1216. my $type = $3;
  1217. die "unknown breaktype $type" unless defined $break_types{$type};
  1218. foreach my $i (hex $1 .. hex $2)
  1219. {
  1220. $break_table[$i] = $break_types{$type};
  1221. }
  1222. next;
  1223. }
  1224. die "malformed line $_";
  1225. }
  1226. close $INPUT;
  1227. open OUTPUT,">$filename.new" or die "Cannot create $filename";
  1228. print "Building $filename\n";
  1229. print OUTPUT "/* Unicode Line Break Properties */\n";
  1230. print OUTPUT "/* generated from $UNIDATA:LineBreak.txt */\n";
  1231. print OUTPUT "/* DO NOT EDIT!! */\n\n";
  1232. print OUTPUT "#include \"windef.h\"\n\n";
  1233. dump_two_level_mapping( "wine_linebreak_table", $break_types{'XX'}, 16, @break_table );
  1234. close OUTPUT;
  1235. save_file($filename);
  1236. }
  1237. my %scripts =
  1238. (
  1239. "Unknown" => 0,
  1240. "Common" => 1,
  1241. "Inherited" => 2,
  1242. "Arabic" => 3,
  1243. "Armenian" => 4,
  1244. "Avestan" => 5,
  1245. "Balinese" => 6,
  1246. "Bamum" => 7,
  1247. "Batak" => 8,
  1248. "Bengali" => 9,
  1249. "Bopomofo" => 10,
  1250. "Brahmi" => 11,
  1251. "Braille" => 12,
  1252. "Buginese" => 13,
  1253. "Buhid" => 14,
  1254. "Canadian_Aboriginal" => 15,
  1255. "Carian" => 16,
  1256. "Cham" => 17,
  1257. "Cherokee" => 18,
  1258. "Coptic" => 19,
  1259. "Cuneiform" => 20,
  1260. "Cypriot" => 21,
  1261. "Cyrillic" => 22,
  1262. "Deseret" => 23,
  1263. "Devanagari" => 24,
  1264. "Egyptian_Hieroglyphs" => 25,
  1265. "Ethiopic" => 26,
  1266. "Georgian" => 27,
  1267. "Glagolitic" => 28,
  1268. "Gothic" => 29,
  1269. "Greek" => 30,
  1270. "Gujarati" => 31,
  1271. "Gurmukhi" => 32,
  1272. "Han" => 33,
  1273. "Hangul" => 34,
  1274. "Hanunoo" => 35,
  1275. "Hebrew" => 36,
  1276. "Hiragana" => 37,
  1277. "Imperial_Aramaic" => 38,
  1278. "Inscriptional_Pahlavi" => 39,
  1279. "Inscriptional_Parthian" => 40,
  1280. "Javanese" => 41,
  1281. "Kaithi" => 42,
  1282. "Kannada" => 43,
  1283. "Katakana" => 44,
  1284. "Kayah_Li" => 45,
  1285. "Kharoshthi" => 46,
  1286. "Khmer" => 47,
  1287. "Lao" => 48,
  1288. "Latin" => 49,
  1289. "Lepcha" => 50,
  1290. "Limbu" => 51,
  1291. "Linear_B" => 52,
  1292. "Lisu" => 53,
  1293. "Lycian" => 54,
  1294. "Lydian" => 55,
  1295. "Malayalam" => 56,
  1296. "Mandaic" => 57,
  1297. "Meetei_Mayek" => 58,
  1298. "Mongolian" => 59,
  1299. "Myanmar" => 60,
  1300. "New_Tai_Lue" => 61,
  1301. "Nko" => 62,
  1302. "Ogham" => 63,
  1303. "Ol_Chiki" => 64,
  1304. "Old_Italic" => 65,
  1305. "Old_Persian" => 66,
  1306. "Old_South_Arabian" => 67,
  1307. "Old_Turkic" => 68,
  1308. "Oriya" => 69,
  1309. "Osmanya" => 70,
  1310. "Phags_Pa" => 71,
  1311. "Phoenician" => 72,
  1312. "Rejang" => 73,
  1313. "Runic" => 74,
  1314. "Samaritan" => 75,
  1315. "Saurashtra" => 76,
  1316. "Shavian" => 77,
  1317. "Sinhala" => 78,
  1318. "Sundanese" => 79,
  1319. "Syloti_Nagri" => 80,
  1320. "Syriac" => 81,
  1321. "Tagalog" => 82,
  1322. "Tagbanwa" => 83,
  1323. "Tai_Le" => 84,
  1324. "Tai_Tham" => 85,
  1325. "Tai_Viet" => 86,
  1326. "Tamil" => 87,
  1327. "Telugu" => 88,
  1328. "Thaana" => 89,
  1329. "Thai" => 90,
  1330. "Tibetan" => 91,
  1331. "Tifinagh" => 92,
  1332. "Ugaritic" => 93,
  1333. "Vai" => 94,
  1334. "Yi" => 95,
  1335. # Win8/Win8.1
  1336. "Chakma" => 96,
  1337. "Meroitic_Cursive" => 97,
  1338. "Meroitic_Hieroglyphs" => 98,
  1339. "Miao" => 99,
  1340. "Sharada" => 100,
  1341. "Sora_Sompeng" => 101,
  1342. "Takri" => 102,
  1343. # Win10
  1344. "Bassa_Vah" => 103,
  1345. "Caucasian_Albanian" => 104,
  1346. "Duployan" => 105,
  1347. "Elbasan" => 106,
  1348. "Grantha" => 107,
  1349. "Khojki" => 108,
  1350. "Khudawadi" => 109,
  1351. "Linear_A" => 110,
  1352. "Mahajani" => 111,
  1353. "Manichaean" => 112,
  1354. "Mende_Kikakui" => 113,
  1355. "Modi" => 114,
  1356. "Mro" => 115,
  1357. "Nabataean" => 116,
  1358. "Old_North_Arabian" => 117,
  1359. "Old_Permic" => 118,
  1360. "Pahawh_Hmong" => 119,
  1361. "Palmyrene" => 120,
  1362. "Pau_Cin_Hau" => 121,
  1363. "Psalter_Pahlavi" => 122,
  1364. "Siddham" => 123,
  1365. "Tirhuta" => 124,
  1366. "Warang_Citi" => 125,
  1367. # Win10 RS1
  1368. "Adlam" => 126,
  1369. "Ahom" => 127,
  1370. "Anatolian_Hieroglyphs" => 128,
  1371. "Bhaiksuki" => 129,
  1372. "Hatran" => 130,
  1373. "Marchen" => 131,
  1374. "Multani" => 132,
  1375. "Newa" => 133,
  1376. "Old_Hungarian" => 134,
  1377. "Osage" => 135,
  1378. "SignWriting" => 136,
  1379. "Tangut" => 137,
  1380. # Win10 RS4
  1381. "Masaram_Gondi" => 138,
  1382. "Nushu" => 139,
  1383. "Soyombo" => 140,
  1384. "Zanabazar_Square" => 141,
  1385. # Win10 1903
  1386. "Dogra" => 142,
  1387. "Gunjala_Gondi" => 143,
  1388. "Hanifi_Rohingya" => 144,
  1389. "Makasar" => 145,
  1390. "Medefaidrin" => 146,
  1391. "Old_Sogdian" => 147,
  1392. "Sogdian" => 148,
  1393. # Win10 2004
  1394. "Elymaic" => 149,
  1395. "Nyiakeng_Puachue_Hmong" => 150,
  1396. "Nandinagari" => 151,
  1397. "Wancho" => 152,
  1398. );
  1399. ################################################################
  1400. # dump Script IDs table
  1401. sub dump_scripts($)
  1402. {
  1403. my $filename = shift;
  1404. my $header = $filename;
  1405. my @scripts_table;
  1406. my $script_index;
  1407. my $i;
  1408. my $INPUT = open_data_file( $UNIDATA, "Scripts.txt" );
  1409. # Fill the table
  1410. # Unknown script id is always 0, so undefined scripts are automatically treated as such
  1411. while (<$INPUT>)
  1412. {
  1413. my $type = "";
  1414. next if /^\#/; # skip comments
  1415. next if /^\s*$/; # skip empty lines
  1416. next if /\x1a/; # skip ^Z
  1417. if (/^\s*([0-9a-fA-F]+)\s*;\s*([a-zA-Z_]+)\s*/)
  1418. {
  1419. $type = $2;
  1420. if (defined $scripts{$type})
  1421. {
  1422. $scripts_table[hex $1] = $scripts{$type};
  1423. }
  1424. next;
  1425. }
  1426. elsif (/^\s*([0-9a-fA-F]+)\.\.\s*([0-9a-fA-F]+)\s*;\s*([a-zA-Z_]+)\s*/)
  1427. {
  1428. $type = $3;
  1429. if (defined $scripts{$type})
  1430. {
  1431. foreach my $i (hex $1 .. hex $2)
  1432. {
  1433. $scripts_table[$i] = $scripts{$type};
  1434. }
  1435. }
  1436. next;
  1437. }
  1438. }
  1439. close $INPUT;
  1440. $header = "$filename.h";
  1441. open OUTPUT,">$header.new" or die "Cannot create $header";
  1442. print "Building $header\n";
  1443. print OUTPUT "/* Unicode Script IDs */\n";
  1444. print OUTPUT "/* generated from $UNIDATA:Scripts.txt */\n";
  1445. print OUTPUT "/* DO NOT EDIT!! */\n\n";
  1446. print OUTPUT "enum unicode_script_id {\n";
  1447. foreach my $script (sort { $scripts{$a} <=> $scripts{$b} } keys %scripts)
  1448. {
  1449. print OUTPUT " Script_$script = $scripts{$script},\n";
  1450. }
  1451. print OUTPUT " Script_LastId = ", (scalar keys %scripts) - 1, "\n";
  1452. print OUTPUT "};\n";
  1453. close OUTPUT;
  1454. save_file($header);
  1455. $filename = "$filename.c";
  1456. open OUTPUT,">$filename.new" or die "Cannot create $header";
  1457. print "Building $filename\n";
  1458. print OUTPUT "/* Unicode Script IDs */\n";
  1459. print OUTPUT "/* generated from $UNIDATA:Scripts.txt */\n";
  1460. print OUTPUT "/* DO NOT EDIT!! */\n\n";
  1461. print OUTPUT "#include \"windef.h\"\n\n";
  1462. dump_two_level_mapping( "wine_scripts_table", 0, 16, @scripts_table );
  1463. close OUTPUT;
  1464. save_file($filename);
  1465. }
  1466. ################################################################
  1467. # dump the BiDi mirroring table
  1468. sub dump_mirroring($)
  1469. {
  1470. my $filename = shift;
  1471. my @mirror_table = ();
  1472. my $INPUT = open_data_file( $UNIDATA, "BidiMirroring.txt" );
  1473. while (<$INPUT>)
  1474. {
  1475. next if /^\#/; # skip comments
  1476. next if /^$/; # skip empty lines
  1477. next if /\x1a/; # skip ^Z
  1478. if (/^\s*([0-9a-fA-F]+)\s*;\s*([0-9a-fA-F]+)/)
  1479. {
  1480. $mirror_table[hex $1] = hex $2;
  1481. next;
  1482. }
  1483. die "malformed line $_";
  1484. }
  1485. close $INPUT;
  1486. open OUTPUT,">$filename.new" or die "Cannot create $filename";
  1487. print "Building $filename\n";
  1488. print OUTPUT "/* Unicode BiDi mirroring */\n";
  1489. print OUTPUT "/* generated from $UNIDATA:BidiMirroring.txt */\n";
  1490. print OUTPUT "/* DO NOT EDIT!! */\n\n";
  1491. print OUTPUT "#include \"windef.h\"\n\n";
  1492. dump_two_level_mapping( "wine_mirror_map", 0, 16, @mirror_table );
  1493. close OUTPUT;
  1494. save_file($filename);
  1495. }
  1496. ################################################################
  1497. # dump the Bidi Brackets
  1498. sub dump_bracket($)
  1499. {
  1500. my $filename = shift;
  1501. my @bracket_table;
  1502. my $INPUT = open_data_file( $UNIDATA, "BidiBrackets.txt" );
  1503. while (<$INPUT>)
  1504. {
  1505. next if /^\#/; # skip comments
  1506. next if /^\s*$/; # skip empty lines
  1507. next if /\x1a/; # skip ^Z
  1508. if (/^\s*([0-9a-fA-F]+)\s*;\s*([0-9a-fA-F]+);\s*([con])/)
  1509. {
  1510. my $type = $3;
  1511. die "unknown bracket $type" unless defined $bracket_types{$type};
  1512. die "characters too distant $1 and $2" if abs(hex($2) - hex($1)) >= 128;
  1513. $bracket_table[hex $1] = (hex($2) - hex($1)) % 255;
  1514. $bracket_table[hex $1] += $bracket_types{$type} << 8;
  1515. next;
  1516. }
  1517. die "malformed line $_";
  1518. }
  1519. close $INPUT;
  1520. open OUTPUT,">$filename.new" or die "Cannot create $filename";
  1521. print "Building $filename\n";
  1522. print OUTPUT "/* Unicode Bidirectional Bracket table */\n";
  1523. print OUTPUT "/* generated from $UNIDATA:BidiBrackets.txt */\n";
  1524. print OUTPUT "/* DO NOT EDIT!! */\n\n";
  1525. print OUTPUT "#include \"windef.h\"\n\n";
  1526. dump_two_level_mapping( "bidi_bracket_table", 0, 16, @bracket_table );
  1527. close OUTPUT;
  1528. save_file($filename);
  1529. }
  1530. ################################################################
  1531. # dump the Arabic shaping table
  1532. sub dump_shaping($)
  1533. {
  1534. my $filename = shift;
  1535. my @joining_table = @initial_joining_table;
  1536. my $INPUT = open_data_file( $UNIDATA, "ArabicShaping.txt" );
  1537. while (<$INPUT>)
  1538. {
  1539. next if /^\#/; # skip comments
  1540. next if /^\s*$/; # skip empty lines
  1541. next if /\x1a/; # skip ^Z
  1542. if (/^\s*([0-9a-fA-F]+)\s*;.*;\s*([RLDCUT])\s*;\s*(\w+)/)
  1543. {
  1544. my $type = $2;
  1545. $joining_table[hex $1] = $joining_types{$type};
  1546. next;
  1547. }
  1548. die "malformed line $_";
  1549. }
  1550. close $INPUT;
  1551. open OUTPUT,">$filename.new" or die "Cannot create $filename";
  1552. print "Building $filename\n";
  1553. print OUTPUT "/* Unicode Arabic shaping */\n";
  1554. print OUTPUT "/* generated from $UNIDATA:ArabicShaping.txt */\n";
  1555. print OUTPUT "/* DO NOT EDIT!! */\n\n";
  1556. print OUTPUT "#include \"windef.h\"\n\n";
  1557. dump_two_level_mapping( "wine_shaping_table", 0, 16, @joining_table );
  1558. print OUTPUT "\nconst unsigned short DECLSPEC_HIDDEN wine_shaping_forms[256][4] =\n{\n";
  1559. for (my $i = 0x600; $i <= 0x6ff; $i++)
  1560. {
  1561. printf OUTPUT " { 0x%04x, 0x%04x, 0x%04x, 0x%04x },\n",
  1562. ${joining_forms{"isolated"}}[$i] || $i,
  1563. ${joining_forms{"final"}}[$i] || $i,
  1564. ${joining_forms{"initial"}}[$i] || $i,
  1565. ${joining_forms{"medial"}}[$i] || $i;
  1566. }
  1567. print OUTPUT "};\n";
  1568. close OUTPUT;
  1569. save_file($filename);
  1570. }
  1571. ################################################################
  1572. # dump the Arabic shaping table
  1573. sub dump_arabic_shaping($)
  1574. {
  1575. my $filename = shift;
  1576. my @joining_table = @initial_joining_table;
  1577. my $INPUT = open_data_file( $UNIDATA, "ArabicShaping.txt" );
  1578. while (<$INPUT>)
  1579. {
  1580. next if /^\#/; # skip comments
  1581. next if /^\s*$/; # skip empty lines
  1582. next if /\x1a/; # skip ^Z
  1583. if (/^\s*([0-9a-fA-F]+)\s*;.*;\s*([RLDCUT])\s*;\s*(\w+)/)
  1584. {
  1585. my $type = $2;
  1586. my $group = $3;
  1587. if ($group eq "ALAPH" || $group eq "DALATH RISH")
  1588. {
  1589. $joining_table[hex $1] = $joining_types{$group};
  1590. }
  1591. else
  1592. {
  1593. $joining_table[hex $1] = $joining_types{$type};
  1594. }
  1595. next;
  1596. }
  1597. die "malformed line $_";
  1598. }
  1599. close $INPUT;
  1600. open OUTPUT,">$filename.new" or die "Cannot create $filename";
  1601. print "Building $filename\n";
  1602. print OUTPUT "/* Unicode Arabic shaping */\n";
  1603. print OUTPUT "/* generated from $UNIDATA:ArabicShaping.txt */\n";
  1604. print OUTPUT "/* DO NOT EDIT!! */\n\n";
  1605. print OUTPUT "#include \"windef.h\"\n\n";
  1606. dump_two_level_mapping( "arabic_shaping_table", 0, 16, @joining_table );
  1607. close OUTPUT;
  1608. save_file($filename);
  1609. }
  1610. ################################################################
  1611. # dump the Vertical Orientation table
  1612. sub dump_vertical($)
  1613. {
  1614. my $filename = shift;
  1615. my @vertical_table;
  1616. my $INPUT = open_data_file( $UNIDATA, "VerticalOrientation.txt" );
  1617. while (<$INPUT>)
  1618. {
  1619. next if /^\#/; # skip comments
  1620. next if /^\s*$/; # skip empty lines
  1621. next if /\x1a/; # skip ^Z
  1622. if (/^\s*([0-9a-fA-F]+)\s*;\s*([a-zA-Z_]+)\s*/)
  1623. {
  1624. my $type = $2;
  1625. die "unknown vertical $type" unless defined $vertical_types{$type};
  1626. if (hex $1 < 65536)
  1627. {
  1628. $vertical_table[hex $1] = $vertical_types{$type};
  1629. }
  1630. next;
  1631. }
  1632. elsif (/^\s*([0-9a-fA-F]+)\.\.\s*([0-9a-fA-F]+)\s*;\s*([A-Za-z_]+)\s*/)
  1633. {
  1634. my $type = $3;
  1635. die "unknown vertical $type" unless defined $vertical_types{$type};
  1636. foreach my $i (hex $1 .. hex $2)
  1637. {
  1638. $vertical_table[$i] = $vertical_types{$type};
  1639. }
  1640. next;
  1641. }
  1642. die "malformed line $_";
  1643. }
  1644. close $INPUT;
  1645. open OUTPUT,">$filename.new" or die "Cannot create $filename";
  1646. print "Building $filename\n";
  1647. print OUTPUT "/* Unicode Vertical Orientation */\n";
  1648. print OUTPUT "/* generated from $UNIDATA:VerticalOrientation.txt */\n";
  1649. print OUTPUT "/* DO NOT EDIT!! */\n\n";
  1650. print OUTPUT "#include \"windef.h\"\n\n";
  1651. dump_two_level_mapping( "vertical_orientation_table", $vertical_types{'R'}, 16, @vertical_table );
  1652. close OUTPUT;
  1653. save_file($filename);
  1654. }
  1655. ################################################################
  1656. # dump the digit folding tables
  1657. sub dump_digit_folding($)
  1658. {
  1659. my ($filename) = shift;
  1660. open OUTPUT,">$filename.new" or die "Cannot create $filename";
  1661. print "Building $filename\n";
  1662. print OUTPUT "/* Unicode digit folding mappings */\n";
  1663. print OUTPUT "/* generated from $UNIDATA:UnicodeData.txt */\n";
  1664. print OUTPUT "/* DO NOT EDIT!! */\n\n";
  1665. print OUTPUT "#include \"windef.h\"\n\n";
  1666. dump_two_level_mapping( "wine_digitmap", 0, 16, @digitmap_table );
  1667. close OUTPUT;
  1668. save_file($filename);
  1669. }
  1670. ################################################################
  1671. # compress a mapping table by removing identical rows
  1672. sub compress_array($$@)
  1673. {
  1674. my $rows = shift;
  1675. my $def = shift;
  1676. my @table = @_;
  1677. my $len = @table / $rows;
  1678. my @array;
  1679. my $data = "";
  1680. # try to merge table rows
  1681. for (my $row = 0; $row < $rows; $row++)
  1682. {
  1683. my $rowtxt = pack "U*", map { defined($_) ? $_ : $def; } @table[($row * $len)..(($row + 1) * $len - 1)];
  1684. my $pos = index $data, $rowtxt;
  1685. if ($pos == -1)
  1686. {
  1687. # check if the tail of the data can match the start of the new row
  1688. my $first = substr( $rowtxt, 0, 1 );
  1689. for (my $i = length($data) - 1; $i > 0; $i--)
  1690. {
  1691. $pos = index( substr( $data, -$i ), $first );
  1692. last if $pos == -1;
  1693. $i -= $pos;
  1694. next unless substr( $data, -$i ) eq substr( $rowtxt, 0, $i );
  1695. substr( $data, -$i ) = "";
  1696. last;
  1697. }
  1698. $pos = length $data;
  1699. $data .= $rowtxt;
  1700. }
  1701. $array[$row] = $rows + $pos;
  1702. }
  1703. return @array, unpack "U*", $data;
  1704. }
  1705. ################################################################
  1706. # dump a char -> 16-bit value mapping table using two-level tables
  1707. sub dump_two_level_mapping($$@)
  1708. {
  1709. my $name = shift;
  1710. my $def = shift;
  1711. my $size = shift;
  1712. my $type = $size == 16 ? "unsigned short" : "unsigned int";
  1713. my @row_array = compress_array( 4096, $def, @_[0..65535] );
  1714. my @array = compress_array( 256, 0, @row_array[0..4095] );
  1715. for (my $i = 256; $i < @array; $i++) { $array[$i] += @array - 4096; }
  1716. printf OUTPUT "const %s DECLSPEC_HIDDEN %s[%d] =\n{\n", $type, $name, @array + @row_array - 4096;
  1717. printf OUTPUT " /* level 1 offsets */\n%s,\n", dump_array( $size, 0, @array[0..255] );
  1718. printf OUTPUT " /* level 2 offsets */\n%s,\n", dump_array( $size, 0, @array[256..$#array] );
  1719. printf OUTPUT " /* values */\n%s\n};\n", dump_array( $size, 0, @row_array[4096..$#row_array] );
  1720. }
  1721. ################################################################
  1722. # dump a char -> value mapping table using three-level tables
  1723. sub dump_three_level_mapping($$@)
  1724. {
  1725. my $name = shift;
  1726. my $def = shift;
  1727. my $size = shift;
  1728. my $type = $size == 16 ? "unsigned short" : "unsigned int";
  1729. my $level3 = ($MAX_CHAR + 1) / 16;
  1730. my $level2 = $level3 / 16;
  1731. my $level1 = $level2 / 16;
  1732. my @array3 = compress_array( $level3, $def, @_[0..$MAX_CHAR] );
  1733. my @array2 = compress_array( $level2, 0, @array3[0..$level3-1] );
  1734. my @array1 = compress_array( $level1, 0, @array2[0..$level2-1] );
  1735. for (my $i = $level2; $i < @array2; $i++) { $array2[$i] += @array1 + @array2 - $level2 - $level3; }
  1736. for (my $i = $level1; $i < @array1; $i++) { $array1[$i] += @array1 - $level2; }
  1737. printf OUTPUT "const %s DECLSPEC_HIDDEN %s[%u] =\n{\n", $type, $name, @array1 + (@array2 - $level2) + (@array3 - $level3);
  1738. printf OUTPUT " /* level 1 offsets */\n%s,\n", dump_array( $size, 0, @array1[0..$level1-1] );
  1739. printf OUTPUT " /* level 2 offsets */\n%s,\n", dump_array( $size, 0, @array1[$level1..$#array1] );
  1740. printf OUTPUT " /* level 3 offsets */\n%s,\n", dump_array( $size, 0, @array2[$level2..$#array2] );
  1741. printf OUTPUT " /* values */\n%s\n};\n", dump_array( $size, 0, @array3[$level3..$#array3] );
  1742. }
  1743. ################################################################
  1744. # dump a binary case mapping table in l_intl.nls format
  1745. sub dump_binary_case_table(@)
  1746. {
  1747. my (@table) = @_;
  1748. my $max_char = 0x10000;
  1749. my $level1 = $max_char / 16;
  1750. my $level2 = $level1 / 16;
  1751. my @difftable;
  1752. for (my $i = 0; $i < @table; $i++)
  1753. {
  1754. next unless defined $table[$i];
  1755. $difftable[$i] = ($table[$i] - $i) & 0xffff;
  1756. }
  1757. my @row_array = compress_array( $level1, 0, @difftable[0..$max_char-1] );
  1758. my @array = compress_array( $level2, 0, @row_array[0..$level1-1] );
  1759. my $offset = @array - $level1;
  1760. for (my $i = $level2; $i < @array; $i++) { $array[$i] += $offset; }
  1761. return pack "S<*", 1 + $offset + @row_array, @array, @row_array[$level1..$#row_array];
  1762. }
  1763. ################################################################
  1764. # dump case mappings for l_intl.nls
  1765. sub dump_intl_nls($)
  1766. {
  1767. my @upper_table = @toupper_table;
  1768. my @lower_table = @tolower_table;
  1769. remove_linguistic_mappings( \@upper_table, \@lower_table );
  1770. my $upper = dump_binary_case_table( @upper_table );
  1771. my $lower = dump_binary_case_table( @lower_table );
  1772. my $filename = shift;
  1773. open OUTPUT,">$filename.new" or die "Cannot create $filename";
  1774. printf "Building $filename\n";
  1775. binmode OUTPUT;
  1776. print OUTPUT pack "S<", 1; # version
  1777. print OUTPUT $upper;
  1778. print OUTPUT $lower;
  1779. close OUTPUT;
  1780. save_file($filename);
  1781. }
  1782. ################################################################
  1783. # dump the bidi direction table
  1784. sub dump_bidi_dir_table($)
  1785. {
  1786. my $filename = shift;
  1787. open OUTPUT,">$filename.new" or die "Cannot create $filename";
  1788. printf "Building $filename\n";
  1789. printf OUTPUT "/* Unicode BiDi direction table */\n";
  1790. printf OUTPUT "/* Automatically generated; DO NOT EDIT!! */\n\n";
  1791. printf OUTPUT "#include \"windef.h\"\n\n";
  1792. my @table;
  1793. for (my $i = 0; $i < 65536; $i++)
  1794. {
  1795. $table[$i] = $bidi_types{$direction_table[$i]} if defined $direction_table[$i];
  1796. }
  1797. dump_two_level_mapping( "bidi_direction_table", $bidi_types{"L"}, 16, @table );
  1798. close OUTPUT;
  1799. save_file($filename);
  1800. }
  1801. sub rol($$)
  1802. {
  1803. my ($byte, $count) = @_;
  1804. return (($byte << $count) | ($byte >> (8 - $count))) & 0xff;
  1805. }
  1806. ################################################################
  1807. # compress the character properties table
  1808. sub compress_char_props_table($@)
  1809. {
  1810. my $rows = shift;
  1811. my @table = @_;
  1812. my $len = @table / $rows;
  1813. my $pos = 0;
  1814. my @array = (0) x $rows;
  1815. my %sequences;
  1816. # add some predefined sequences
  1817. foreach my $i (0, 0xfb .. 0xff) { $sequences{pack "L*", (rol($i,5)) x $len} = $i; }
  1818. # try to merge table rows
  1819. for (my $row = 0; $row < $rows; $row++)
  1820. {
  1821. my @table_row = map { defined $_ ? $_ : 0x7f; } @table[($row * $len)..(($row + 1) * $len - 1)];
  1822. my $rowtxt = pack "L*", @table_row;
  1823. if (defined($sequences{$rowtxt}))
  1824. {
  1825. # reuse an existing row
  1826. $array[$row] = $sequences{$rowtxt};
  1827. }
  1828. else
  1829. {
  1830. # create a new row
  1831. $sequences{$rowtxt} = $array[$row] = ++$pos;
  1832. push @array, @table_row;
  1833. }
  1834. }
  1835. return @array;
  1836. }
  1837. ################################################################
  1838. # dump a normalization table in binary format
  1839. sub dump_norm_table($)
  1840. {
  1841. my $filename = shift;
  1842. my %forms = ( "nfc" => 1, "nfd" => 2, "nfkc" => 5, "nfkd" => 6, "idna" => 13 );
  1843. my %decomp = ( "nfc" => \@decomp_table,
  1844. "nfd" => \@decomp_table,
  1845. "nfkc" => \@decomp_compat_table,
  1846. "nfkd" => \@decomp_compat_table ,
  1847. "idna" => \@idna_decomp_table );
  1848. open OUTPUT,">$filename.new" or die "Cannot create $filename";
  1849. print "Building $filename\n";
  1850. my $type = $filename;
  1851. $type =~ s!.*/norm(\w+)\.nls!$1!;
  1852. my $compose = $forms{$type} & 1;
  1853. my $compat = !!($forms{$type} & 4) + ($type eq "idna");
  1854. my @version = split /\./, $UNIVERSION;
  1855. # combining classes
  1856. my @classes;
  1857. my @class_values;
  1858. foreach my $c (grep defined, @combining_class_table)
  1859. {
  1860. $classes[$c] = 1 if $c < 0x100;
  1861. }
  1862. for (my $i = 0; $i < @classes; $i++)
  1863. {
  1864. next unless defined $classes[$i];
  1865. $classes[$i] = @class_values;
  1866. push @class_values, $i;
  1867. }
  1868. push @class_values, 0 if (@class_values % 2);
  1869. die "too many classes" if @class_values >= 0x40;
  1870. # character properties
  1871. my @char_props;
  1872. my @decomposed;
  1873. my @comp_hash_table;
  1874. my $comp_hash_size = $compose ? 254 : 0;
  1875. for (my $i = 0; $i <= $MAX_CHAR; $i++)
  1876. {
  1877. next unless defined $combining_class_table[$i];
  1878. if (defined $decomp{$type}->[$i])
  1879. {
  1880. my @dec = get_decomposition( $i, $decomp{$type} );
  1881. if ($compose && (my @comp = get_composition( $i, $compat )))
  1882. {
  1883. my $hash = ($comp[0] + 95 * $comp[1]) % $comp_hash_size;
  1884. push @{$comp_hash_table[$hash]}, to_utf16( @comp, $i );
  1885. my $val = 0;
  1886. foreach my $d (@dec)
  1887. {
  1888. $val = $combining_class_table[$d];
  1889. last if $val;
  1890. }
  1891. $char_props[$i] = $classes[$val];
  1892. }
  1893. else
  1894. {
  1895. $char_props[$i] = 0xbf;
  1896. }
  1897. @dec = compose_hangul( @dec ) if $compose;
  1898. @dec = to_utf16( @dec );
  1899. push @dec, 0 if @dec >= 7;
  1900. $decomposed[$i] = \@dec;
  1901. }
  1902. else
  1903. {
  1904. if ($combining_class_table[$i] == 0x100)
  1905. {
  1906. $char_props[$i] = 0x7f;
  1907. }
  1908. elsif ($combining_class_table[$i])
  1909. {
  1910. $char_props[$i] = $classes[$combining_class_table[$i]] | 0x80;
  1911. }
  1912. elsif ($type eq "idna" && defined $idna_disallowed[$i])
  1913. {
  1914. $char_props[$i] = 0xff;
  1915. }
  1916. else
  1917. {
  1918. $char_props[$i] = 0;
  1919. }
  1920. }
  1921. }
  1922. if ($compose)
  1923. {
  1924. for (my $i = 0; $i <= $MAX_CHAR; $i++)
  1925. {
  1926. my @comp = get_composition( $i, $compat );
  1927. next unless @comp;
  1928. if ($combining_class_table[$comp[1]])
  1929. {
  1930. $char_props[$comp[0]] |= 0x40 unless $char_props[$comp[0]] & 0x80;
  1931. $char_props[$comp[1]] |= 0x40;
  1932. }
  1933. else
  1934. {
  1935. $char_props[$comp[0]] = ($char_props[$comp[0]] & ~0x40) | 0x80;
  1936. $char_props[$comp[1]] |= 0xc0;
  1937. }
  1938. }
  1939. }
  1940. # surrogates
  1941. foreach my $i (0xd800..0xdbff) { $char_props[$i] = 0xdf; }
  1942. foreach my $i (0xdc00..0xdfff) { $char_props[$i] = 0x9f; }
  1943. # Hangul
  1944. if ($type eq "nfc") { foreach my $i (0x1100..0x117f) { $char_props[$i] = 0xff; } }
  1945. elsif ($compose) { foreach my $i (0x1100..0x11ff) { $char_props[$i] = 0xff; } }
  1946. foreach my $i (0xac00..0xd7ff) { $char_props[$i] = 0xff; }
  1947. # invalid chars
  1948. if ($type eq "idna") { foreach my $i (0x00..0x1f, 0x7f) { $char_props[$i] = 0xff; } }
  1949. foreach my $i (0xfdd0..0xfdef) { $char_props[$i] = 0xff; }
  1950. foreach my $i (0x00..0x10)
  1951. {
  1952. $char_props[($i << 16) | 0xfffe] = 0xff;
  1953. $char_props[($i << 16) | 0xffff] = 0xff;
  1954. }
  1955. # decomposition hash table
  1956. my @decomp_hash_table;
  1957. my @decomp_hash_index;
  1958. my @decomp_hash_data;
  1959. my $decomp_hash_size = 944;
  1960. # build string of character data, reusing substrings when possible
  1961. my $decomp_char_data = "";
  1962. foreach my $i (sort { @{$b} <=> @{$a} } grep defined, @decomposed)
  1963. {
  1964. my $str = pack "U*", @{$i};
  1965. $decomp_char_data .= $str if index( $decomp_char_data, $str) == -1;
  1966. }
  1967. for (my $i = 0; $i < @decomposed; $i++)
  1968. {
  1969. next unless defined $decomposed[$i];
  1970. my $pos = index( $decomp_char_data, pack( "U*", @{$decomposed[$i]} ));
  1971. die "sequence not found" if $pos == -1;
  1972. my $len = @{$decomposed[$i]};
  1973. $len = 7 if $len > 7;
  1974. my $hash = $i % $decomp_hash_size;
  1975. push @{$decomp_hash_table[$hash]}, [ $i, ($len << 13) | $pos ];
  1976. }
  1977. for (my $i = 0; $i < $decomp_hash_size; $i++)
  1978. {
  1979. $decomp_hash_index[$i] = @decomp_hash_data / 2;
  1980. next unless defined $decomp_hash_table[$i];
  1981. if (@{$decomp_hash_table[$i]} == 1)
  1982. {
  1983. my $entry = $decomp_hash_table[$i]->[0];
  1984. if ($char_props[$entry->[0]] == 0xbf)
  1985. {
  1986. $decomp_hash_index[$i] = $entry->[1];
  1987. next;
  1988. }
  1989. }
  1990. foreach my $entry (@{$decomp_hash_table[$i]})
  1991. {
  1992. push @decomp_hash_data, $entry->[0] & 0xffff, $entry->[1];
  1993. }
  1994. }
  1995. push @decomp_hash_data, 0, 0;
  1996. # composition hash table
  1997. my @comp_hash_index;
  1998. my @comp_hash_data;
  1999. if (@comp_hash_table)
  2000. {
  2001. for (my $i = 0; $i < $comp_hash_size; $i++)
  2002. {
  2003. $comp_hash_index[$i] = @comp_hash_data;
  2004. push @comp_hash_data, @{$comp_hash_table[$i]} if defined $comp_hash_table[$i];
  2005. }
  2006. $comp_hash_index[$comp_hash_size] = @comp_hash_data;
  2007. push @comp_hash_data, 0, 0, 0;
  2008. }
  2009. my $level1 = ($MAX_CHAR + 1) / 128;
  2010. my @rows = compress_char_props_table( $level1, @char_props[0..$MAX_CHAR] );
  2011. my @header = ( $version[0], $version[1], $version[2], 0, $forms{$type}, $compat ? 18 : 3,
  2012. 0, $decomp_hash_size, $comp_hash_size, 0 );
  2013. my @tables = (0) x 8;
  2014. $tables[0] = 16 + @header + @tables;
  2015. $tables[1] = $tables[0] + @class_values / 2;
  2016. $tables[2] = $tables[1] + $level1 / 2;
  2017. $tables[3] = $tables[2] + (@rows - $level1) / 2;
  2018. $tables[4] = $tables[3] + @decomp_hash_index;
  2019. $tables[5] = $tables[4] + @decomp_hash_data;
  2020. $tables[6] = $tables[5] + length $decomp_char_data;
  2021. $tables[7] = $tables[6] + @comp_hash_index;
  2022. print OUTPUT pack "S<16", unpack "U*", "norm$type.nlp";
  2023. print OUTPUT pack "S<*", @header;
  2024. print OUTPUT pack "S<*", @tables;
  2025. print OUTPUT pack "C*", @class_values;
  2026. print OUTPUT pack "C*", @rows[0..$level1-1];
  2027. print OUTPUT pack "C*", @rows[$level1..$#rows];
  2028. print OUTPUT pack "S<*", @decomp_hash_index;
  2029. print OUTPUT pack "S<*", @decomp_hash_data;
  2030. print OUTPUT pack "S<*", unpack "U*", $decomp_char_data;
  2031. print OUTPUT pack "S<*", @comp_hash_index;
  2032. print OUTPUT pack "S<*", @comp_hash_data;
  2033. close OUTPUT;
  2034. save_file($filename);
  2035. add_registry_value( "Normalization", sprintf( "%x", $forms{$type} ), "norm$type.nls" );
  2036. }
  2037. ################################################################
  2038. # output a codepage definition file from the global tables
  2039. sub output_codepage_file($)
  2040. {
  2041. my $codepage = shift;
  2042. my $output = sprintf "nls/c_%03d.nls", $codepage;
  2043. open OUTPUT,">$output.new" or die "Cannot create $output";
  2044. printf "Building %s\n", $output;
  2045. if (!@lead_bytes) { dump_binary_sbcs_table( $codepage ); }
  2046. else { dump_binary_dbcs_table( $codepage ); }
  2047. close OUTPUT;
  2048. save_file($output);
  2049. add_registry_value( "Codepage", sprintf( "%d", $codepage ), sprintf( "c_%03d.nls", $codepage ));
  2050. }
  2051. ################################################################
  2052. # output a codepage table from a Microsoft-style mapping file
  2053. sub dump_msdata_codepage($)
  2054. {
  2055. my $filename = shift;
  2056. my $state = "";
  2057. my ($codepage, $width, $count);
  2058. my ($lb_cur, $lb_end);
  2059. @cp2uni = ();
  2060. @glyph2uni = ();
  2061. @lead_bytes = ();
  2062. @uni2cp = ();
  2063. $default_char = $DEF_CHAR;
  2064. $default_wchar = $DEF_CHAR;
  2065. my $INPUT = open_data_file( $MSCODEPAGES, $filename ) or die "Cannot open $filename";
  2066. while (<$INPUT>)
  2067. {
  2068. next if /^;/; # skip comments
  2069. next if /^\s*$/; # skip empty lines
  2070. next if /\x1a/; # skip ^Z
  2071. last if /^ENDCODEPAGE/;
  2072. if (/^CODEPAGE\s+(\d+)/)
  2073. {
  2074. $codepage = $1;
  2075. next;
  2076. }
  2077. if (/^CPINFO\s+(\d+)\s+0x([0-9a-fA-f]+)\s+0x([0-9a-fA-F]+)/)
  2078. {
  2079. $width = $1;
  2080. $default_char = hex $2;
  2081. $default_wchar = hex $3;
  2082. next;
  2083. }
  2084. if (/^(MBTABLE|GLYPHTABLE|WCTABLE|DBCSRANGE|DBCSTABLE)\s+(\d+)/)
  2085. {
  2086. $state = $1;
  2087. $count = $2;
  2088. next;
  2089. }
  2090. if (/^0x([0-9a-fA-F]+)\s+0x([0-9a-fA-F]+)/)
  2091. {
  2092. if ($state eq "MBTABLE")
  2093. {
  2094. my $cp = hex $1;
  2095. my $uni = hex $2;
  2096. $cp2uni[$cp] = $uni unless defined($cp2uni[$cp]);
  2097. next;
  2098. }
  2099. if ($state eq "GLYPHTABLE")
  2100. {
  2101. my $cp = hex $1;
  2102. my $uni = hex $2;
  2103. $glyph2uni[$cp] = $uni unless defined($glyph2uni[$cp]);
  2104. next;
  2105. }
  2106. if ($state eq "WCTABLE")
  2107. {
  2108. my $uni = hex $1;
  2109. my $cp = hex $2;
  2110. $uni2cp[$uni] = $cp unless defined($uni2cp[$uni]);
  2111. next;
  2112. }
  2113. if ($state eq "DBCSRANGE")
  2114. {
  2115. my $start = hex $1;
  2116. my $end = hex $2;
  2117. for (my $i = $start; $i <= $end; $i++) { add_lead_byte( $i ); }
  2118. $lb_cur = $start;
  2119. $lb_end = $end;
  2120. next;
  2121. }
  2122. if ($state eq "DBCSTABLE")
  2123. {
  2124. my $mb = hex $1;
  2125. my $uni = hex $2;
  2126. my $cp = ($lb_cur << 8) | $mb;
  2127. $cp2uni[$cp] = $uni unless defined($cp2uni[$cp]);
  2128. if (!--$count)
  2129. {
  2130. if (++$lb_cur > $lb_end) { $state = "DBCSRANGE"; }
  2131. }
  2132. next;
  2133. }
  2134. }
  2135. die "$filename: Unrecognized line $_\n";
  2136. }
  2137. close $INPUT;
  2138. output_codepage_file( $codepage );
  2139. if ($codepage == 949) { dump_krwansung_codepage( @uni2cp ); }
  2140. }
  2141. ################################################################
  2142. # align a string length
  2143. sub align_string($$)
  2144. {
  2145. my ($align, $str) = @_;
  2146. $str .= pack "C*", (0) x ($align - length($str) % $align) if length($str) % $align;
  2147. return $str;
  2148. }
  2149. ################################################################
  2150. # pack a GUID string
  2151. sub pack_guid($)
  2152. {
  2153. $_ = shift;
  2154. /([0-9A-Fa-f]{8})-([0-9A-Fa-f]{4})-([0-9A-Fa-f]{4})-([0-9A-Fa-f]{2})([0-9A-Fa-f]{2})-([0-9A-Fa-f]{2})([0-9A-Fa-f]{2})([0-9A-Fa-f]{2})([0-9A-Fa-f]{2})([0-9A-Fa-f]{2})([0-9A-Fa-f]{2})/;
  2155. return pack "L<S<2C8", hex $1, hex $2, hex $3, hex $4, hex $5, hex $6, hex $7, hex $8, hex $9, hex $10, hex $11;
  2156. }
  2157. ################################################################
  2158. # comparison function for compression sort
  2159. sub cmp_compression
  2160. {
  2161. return scalar @{$a} <=> scalar @{$b} ||
  2162. $a->[4] <=> $b->[4] ||
  2163. $a->[5] <=> $b->[5] ||
  2164. $a->[6] <=> $b->[6] ||
  2165. $a->[7] <=> $b->[7] ||
  2166. $a->[8] <=> $b->[8] ||
  2167. $a->[9] <=> $b->[9] ||
  2168. $a->[10] <=> $b->[10] ||
  2169. $a->[11] <=> $b->[11] ||
  2170. $a->[12] <=> $b->[12];
  2171. }
  2172. ################################################################
  2173. # build a binary sort keys table
  2174. sub dump_sortkey_table($$)
  2175. {
  2176. my ($filename, $download) = @_;
  2177. my @keys;
  2178. my ($part, $section, $subsection, $guid, $version, $ling_flag);
  2179. my @multiple_weights;
  2180. my @expansions;
  2181. my @compressions;
  2182. my %exceptions;
  2183. my %guids;
  2184. my %compr_flags;
  2185. my %locales;
  2186. my $default_guid = "00000001-57ee-1e5c-00b4-d0000bb1e11e";
  2187. my $jamostr = "";
  2188. my $re_hex = '0x[0-9A-Fa-f]+';
  2189. my $re_key = '(\d+\s+\d+\s+\d+\s+\d+)';
  2190. $guids{$default_guid} = { };
  2191. my %flags = ( "HAS_3_BYTE_WEIGHTS" => 0x01, "REVERSEDIACRITICS" => 0x10, "DOUBLECOMPRESSION" => 0x20, "INVERSECASING" => 0x40 );
  2192. my $KEYS = open_data_file( $MSDATA, $download );
  2193. printf "Building $filename\n";
  2194. while (<$KEYS>)
  2195. {
  2196. s/\s*;.*$//;
  2197. next if /^\s*$/; # skip empty lines
  2198. if (/^\s*(SORTKEY|SORTTABLES)/)
  2199. {
  2200. $part = $1;
  2201. next;
  2202. }
  2203. if (/^\s*(ENDSORTKEY|ENDSORTTABLES)/)
  2204. {
  2205. $part = $section = "";
  2206. next;
  2207. }
  2208. if (/^\s*(DEFAULT|RELEASE|REVERSEDIACRITICS|DOUBLECOMPRESSION|INVERSECASING|MULTIPLEWEIGHTS|EXPANSION|COMPATIBILITY|COMPRESSION|EXCEPTION|JAMOSORT)\s+/)
  2209. {
  2210. $section = $1;
  2211. $guid = undef;
  2212. next;
  2213. }
  2214. next unless $part;
  2215. if ("$part.$section" eq "SORTKEY.DEFAULT")
  2216. {
  2217. if (/^\s*($re_hex)\s+$re_key/)
  2218. {
  2219. $keys[hex $1] = [ split(/\s+/,$2) ];
  2220. next;
  2221. }
  2222. }
  2223. elsif ("$part.$section" eq "SORTTABLES.RELEASE")
  2224. {
  2225. if (/^\s*NLSVERSION\s+0x([0-9A-Fa-f]+)/)
  2226. {
  2227. $version = hex $1;
  2228. next;
  2229. }
  2230. if (/^\s*DEFINEDVERSION\s+0x([0-9A-Fa-f]+)/)
  2231. {
  2232. # ignore for now
  2233. next;
  2234. }
  2235. }
  2236. elsif ("$part.$section" eq "SORTTABLES.REVERSEDIACRITICS" ||
  2237. "$part.$section" eq "SORTTABLES.DOUBLECOMPRESSION" ||
  2238. "$part.$section" eq "SORTTABLES.INVERSECASING")
  2239. {
  2240. if (/^\s*SORTGUID\s+([-0-9A-Fa-f]+)/)
  2241. {
  2242. $guid = lc $1;
  2243. $guids{$guid} = { } unless defined $guids{$guid};
  2244. $guids{$guid}->{flags} |= $flags{$section};
  2245. next;
  2246. }
  2247. if (/^\s*LOCALENAME\s+([A-Za-z0-9-_]+)/)
  2248. {
  2249. $locales{$1} = $guid;
  2250. next;
  2251. }
  2252. }
  2253. elsif ("$part.$section" eq "SORTTABLES.MULTIPLEWEIGHTS")
  2254. {
  2255. if (/^\s*(\d+)\s+(\d+)/)
  2256. {
  2257. push @multiple_weights, $1, $2;
  2258. next;
  2259. }
  2260. }
  2261. elsif ("$part.$section" eq "SORTTABLES.EXPANSION")
  2262. {
  2263. if (/^\s*0x([0-9A-Fa-f]+)\s+0x([0-9A-Fa-f]+)\s+0x([0-9A-Fa-f]+)/)
  2264. {
  2265. my $pos = scalar @expansions / 2;
  2266. $keys[hex $1] = [ 2, 0, $pos & 0xff, $pos >> 8 ] unless defined $keys[hex $1];
  2267. push @expansions, hex $2, hex $3;
  2268. next;
  2269. }
  2270. }
  2271. elsif ("$part.$section" eq "SORTTABLES.COMPATIBILITY")
  2272. {
  2273. if (/^\s*0x([0-9A-Fa-f]+)\s+0x([0-9A-Fa-f]+)/)
  2274. {
  2275. $keys[hex $1] = $keys[hex $2];
  2276. next;
  2277. }
  2278. }
  2279. elsif ("$part.$section" eq "SORTTABLES.COMPRESSION")
  2280. {
  2281. if (/^\s*SORTGUID\s+([-0-9A-Fa-f]+)\s+\d*\s*([A-Z0-9_]+)?/)
  2282. {
  2283. if ($subsection || !$guid) # start a new one
  2284. {
  2285. $guid = lc $1;
  2286. $subsection = "";
  2287. $guids{$guid} = { } unless defined $guids{$guid};
  2288. $guids{$guid}->{flags} |= $flags{$2} if $2;
  2289. $guids{$guid}->{compr} = @compressions;
  2290. $exceptions{"$guid-"} = [ ] unless defined $exceptions{"$guid-"};
  2291. $compr_flags{$guid} = [ ] unless defined $compr_flags{$guid};
  2292. push @compressions, [ ];
  2293. }
  2294. else # merge with current one
  2295. {
  2296. $guids{lc $1} = { } unless defined $guids{lc $1};
  2297. $guids{lc $1}->{flags} |= $flags{$2} if $2;
  2298. $guids{lc $1}->{compr} = $guids{$guid}->{compr};
  2299. $compr_flags{lc $1} = $compr_flags{$guid};
  2300. }
  2301. next;
  2302. }
  2303. if (/^\s*LOCALENAME\s+([A-Za-z0-9-_]+)/)
  2304. {
  2305. $locales{$1} = $guid;
  2306. next;
  2307. }
  2308. if (/^\s*(TWO|THREE|FOUR|FIVE|SIX|SEVEN|EIGHT)/)
  2309. {
  2310. $subsection = $1;
  2311. next;
  2312. }
  2313. if ($subsection && /^\s*(($re_hex\s+){2,8})$re_key/)
  2314. {
  2315. my @comp = map { hex $_; } split(/\s+/,$1);
  2316. push @{$compressions[$#compressions]}, [ split(/\s+/,$3), @comp ];
  2317. # add compression flags
  2318. $compr_flags{$guid}->[$comp[0]] |= @comp >= 6 ? 0xc0 : @comp >= 4 ? 0x80 : 0x40;
  2319. next;
  2320. }
  2321. }
  2322. elsif ("$part.$section" eq "SORTTABLES.EXCEPTION")
  2323. {
  2324. if (/^\s*SORTGUID\s+([-0-9A-Fa-f]+)\s+\d*\s*(LINGUISTIC_CASING)?/)
  2325. {
  2326. $guid = lc $1;
  2327. $guids{$guid} = { } unless defined $guids{lc $1};
  2328. $ling_flag = ($2 ? "+" : "-");
  2329. $exceptions{"$guid$ling_flag"} = [ ] unless defined $exceptions{"$guid$ling_flag"};
  2330. next;
  2331. }
  2332. if (/^\s*LOCALENAME\s+([A-Za-z0-9-_]+)/)
  2333. {
  2334. $locales{$1} = $guid;
  2335. next;
  2336. }
  2337. if (/^\s*($re_hex)\s+$re_key/)
  2338. {
  2339. $exceptions{"$guid$ling_flag"}->[hex $1] = [ split(/\s+/,$2) ];
  2340. next;
  2341. }
  2342. }
  2343. elsif ("$part.$section" eq "SORTTABLES.JAMOSORT")
  2344. {
  2345. if (/^\s*$re_hex\s+(($re_hex\s*){5})/)
  2346. {
  2347. $jamostr .= pack "C8", map { hex $_; } split /\s+/, $1;
  2348. next;
  2349. }
  2350. }
  2351. die "$download: $part.$section: unrecognized line $_\n";
  2352. }
  2353. close $KEYS;
  2354. # Sortkey table
  2355. my $table;
  2356. for (my $i = 0; $i < 0x10000; $i++)
  2357. {
  2358. my @k = defined $keys[$i] ? @{$keys[$i]} : (0) x 4;
  2359. $table .= pack "C4", $k[1], $k[0], $k[2], $k[3];
  2360. }
  2361. foreach my $id (sort keys %exceptions)
  2362. {
  2363. my $pos = length($table) / 4;
  2364. my @exc = @{$exceptions{$id}};
  2365. my @filled;
  2366. my $key = (substr( $id, -1 ) eq "+" ? "ling_except" : "except");
  2367. my $guid = substr( $id, 0, -1 );
  2368. $guids{$guid}->{$key} = $pos;
  2369. $pos += 0x100;
  2370. my @flags = @{$compr_flags{$guid}} if defined $compr_flags{$guid};
  2371. for (my $j = 0; $j < 0x10000; $j++)
  2372. {
  2373. next unless defined $exc[$j] || defined $flags[$j];
  2374. $filled[$j >> 8] = 1;
  2375. $j |= 0xff;
  2376. }
  2377. for (my $j = 0; $j < 0x100; $j++)
  2378. {
  2379. $table .= pack "L<", $filled[$j] ? $pos : $j * 0x100;
  2380. $pos += 0x100 if $filled[$j];
  2381. }
  2382. for (my $j = 0; $j < 0x10000; $j++)
  2383. {
  2384. next unless $filled[$j >> 8];
  2385. my @k = defined $exc[$j] ? @{$exc[$j]} : defined $keys[$j] ? @{$keys[$j]} : (0) x 4;
  2386. $k[3] |= $flags[$j] || 0;
  2387. $table .= pack "C4", $k[1], $k[0], $k[2], $k[3];
  2388. }
  2389. }
  2390. # Case mapping tables
  2391. # standard table
  2392. my @casemaps;
  2393. my @upper = @toupper_table;
  2394. my @lower = @tolower_table;
  2395. remove_linguistic_mappings( \@upper, \@lower );
  2396. $casemaps[0] = pack( "S<*", 1) . dump_binary_case_table( @upper ) . dump_binary_case_table( @lower );
  2397. # linguistic table
  2398. $casemaps[1] = pack( "S<*", 1) . dump_binary_case_table( @toupper_table ) . dump_binary_case_table( @tolower_table );
  2399. # Turkish table
  2400. @upper = @toupper_table;
  2401. @lower = @tolower_table;
  2402. $upper[ord 'i'] = 0x130; # LATIN CAPITAL LETTER I WITH DOT ABOVE
  2403. $lower[ord 'I'] = 0x131; # LATIN SMALL LETTER DOTLESS I
  2404. $casemaps[2] = pack( "S<*", 1) . dump_binary_case_table( @upper ) . dump_binary_case_table( @lower );
  2405. my $casemaps = align_string( 8, $casemaps[0] . $casemaps[1] . $casemaps[2] );
  2406. # Char type table
  2407. my @table;
  2408. my $types = "";
  2409. my %typestr;
  2410. for (my $i = 0; $i < 0x10000; $i++)
  2411. {
  2412. my $str = pack "S<3",
  2413. ($category_table[$i] || 0) & 0xffff,
  2414. defined($direction_table[$i]) ? $c2_types{$direction_table[$i]} : 0,
  2415. ($category_table[$i] || 0) >> 16;
  2416. if (!defined($typestr{$str}))
  2417. {
  2418. $typestr{$str} = length($types) / 6;
  2419. $types .= $str;
  2420. }
  2421. $table[$i] = $typestr{$str};
  2422. }
  2423. my @rows = compress_array( 4096, 0, @table[0..65535] );
  2424. my @array = compress_array( 256, 0, @rows[0..4095] );
  2425. for (my $i = 0; $i < 256; $i++) { $array[$i] *= 2; } # we need byte offsets
  2426. for (my $i = 256; $i < @array; $i++) { $array[$i] += 2 * @array - 4096; }
  2427. my $arraystr = pack("S<*", @array) . pack("C*", @rows[4096..$#rows]);
  2428. my $chartypes = pack "S<2", 4 + length($types) + length($arraystr), 2 + length($types);
  2429. $chartypes = align_string( 8, $chartypes . $types . $arraystr );
  2430. # Sort tables
  2431. # guids
  2432. my $sorttables = pack "L<2", $version, scalar %guids;
  2433. foreach my $id (sort keys %guids)
  2434. {
  2435. my %guid = %{$guids{$id}};
  2436. my $flags = $guid{flags} || 0;
  2437. my $map = length($casemaps[0]) + (defined $guid{ling_except} ? length($casemaps[1]) : 0);
  2438. $sorttables .= pack_guid($id) . pack "L<5",
  2439. $flags,
  2440. defined($guid{compr}) ? $guid{compr} : 0xffffffff,
  2441. $guid{except} || 0,
  2442. $guid{ling_except} || 0,
  2443. $map / 2;
  2444. }
  2445. # expansions
  2446. $sorttables .= pack "L<S<*", scalar @expansions / 2, @expansions;
  2447. # compressions
  2448. $sorttables .= pack "L<", scalar @compressions;
  2449. my $rowstr = "";
  2450. foreach my $c (@compressions)
  2451. {
  2452. my $pos = length($rowstr) / 2;
  2453. my $min = 0xffff;
  2454. my $max = 0;
  2455. my @lengths = (0) x 8;
  2456. foreach my $r (sort cmp_compression @{$c})
  2457. {
  2458. my @row = @{$r};
  2459. $lengths[scalar @row - 6]++;
  2460. foreach my $val (@row[4..$#row])
  2461. {
  2462. $min = $val if $min > $val;
  2463. $max = $val if $max < $val;
  2464. }
  2465. $rowstr .= align_string( 4, pack "S<*", @row[4..$#row] );
  2466. $rowstr .= pack "C4", $row[1], $row[0], $row[2], $row[3];
  2467. }
  2468. $sorttables .= pack "L<S<10", $pos, $min, $max, @lengths;
  2469. }
  2470. $sorttables .= $rowstr;
  2471. # multiple weights
  2472. $sorttables .= align_string( 4, pack "L<C*", scalar @multiple_weights / 2, @multiple_weights );
  2473. # jamo sort
  2474. $sorttables .= pack("L<", length($jamostr) / 8) . $jamostr;
  2475. # Locales
  2476. add_registry_key( "Sorting\\Ids", "{$default_guid}" );
  2477. foreach my $loc (sort keys %locales)
  2478. {
  2479. # skip specific locales that match more general ones
  2480. my @parts = split /[-_]/, $loc;
  2481. next if @parts > 1 && defined($locales{$parts[0]}) && $locales{$parts[0]} eq $locales{$loc};
  2482. next if @parts > 2 && defined($locales{"$parts[0]-$parts[1]"}) && $locales{"$parts[0]-$parts[1]"} eq $locales{$loc};
  2483. add_registry_value( "Sorting\\Ids", $loc, "\{$locales{$loc}\}" );
  2484. }
  2485. # File header
  2486. my @header;
  2487. $header[0] = 16;
  2488. $header[1] = $header[0] + length $table;
  2489. $header[2] = $header[1] + length $casemaps;
  2490. $header[3] = $header[2] + length $chartypes;
  2491. open OUTPUT, ">$filename.new" or die "Cannot create $filename";
  2492. print OUTPUT pack "L<*", @header;
  2493. print OUTPUT $table, $casemaps, $chartypes, $sorttables;
  2494. close OUTPUT;
  2495. save_file($filename);
  2496. }
  2497. ################################################################
  2498. # build the script to create registry keys
  2499. sub dump_registry_script($%)
  2500. {
  2501. my ($filename, %keys) = @_;
  2502. my $indent = 1;
  2503. printf "Building %s\n", $filename;
  2504. open OUTPUT, ">$filename.new" or die "Cannot create $filename";
  2505. print OUTPUT "HKLM\n{\n";
  2506. foreach my $k (split /\\/, "SYSTEM\\CurrentControlSet\\Control\\Nls")
  2507. {
  2508. printf OUTPUT "%*sNoRemove %s\n%*s{\n", 4 * $indent, "", $k, 4 * $indent, "";
  2509. $indent++;
  2510. }
  2511. foreach my $k (sort keys %keys)
  2512. {
  2513. my @subkeys = split /\\/, $k;
  2514. my ($def, @vals) = @{$keys{$k}};
  2515. for (my $i = 0; $i < @subkeys; $i++)
  2516. {
  2517. printf OUTPUT "%*s%s%s\n%*s{\n", 4 * $indent, "", $subkeys[$i],
  2518. $i == $#subkeys && $def ? " = s '$def'" : "", 4 * $indent, "";
  2519. $indent++;
  2520. }
  2521. foreach my $v (sort @vals) { printf OUTPUT "%*sval $v\n", 4 * $indent, ""; }
  2522. for (my $i = 0; $i < @subkeys; $i++) { printf OUTPUT "%*s}\n", 4 * --$indent, ""; }
  2523. }
  2524. while ($indent) { printf OUTPUT "%*s}\n", 4 * --$indent, ""; }
  2525. close OUTPUT;
  2526. save_file($filename);
  2527. }
  2528. ################################################################
  2529. # save a file if modified
  2530. sub save_file($)
  2531. {
  2532. my $file = shift;
  2533. if (-f $file && !system "cmp $file $file.new >/dev/null")
  2534. {
  2535. unlink "$file.new";
  2536. }
  2537. else
  2538. {
  2539. rename "$file.new", "$file";
  2540. }
  2541. }
  2542. ################################################################
  2543. # main routine
  2544. chdir ".." if -f "./make_unicode";
  2545. load_data();
  2546. dump_sortkeys( "dlls/kernelbase/collation.c" );
  2547. dump_bidi_dir_table( "dlls/gdi32/uniscribe/direction.c" );
  2548. dump_bidi_dir_table( "dlls/dwrite/direction.c" );
  2549. dump_digit_folding( "dlls/kernelbase/digitmap.c" );
  2550. dump_mirroring( "dlls/gdi32/uniscribe/mirror.c" );
  2551. dump_mirroring( "dlls/dwrite/mirror.c" );
  2552. dump_bracket( "dlls/gdi32/uniscribe/bracket.c" );
  2553. dump_bracket( "dlls/dwrite/bracket.c" );
  2554. dump_shaping( "dlls/gdi32/uniscribe/shaping.c" );
  2555. dump_arabic_shaping( "dlls/dwrite/shapers/arabic_table.c" );
  2556. dump_linebreak( "dlls/gdi32/uniscribe/linebreak.c" );
  2557. dump_linebreak( "dlls/dwrite/linebreak.c" );
  2558. dump_scripts( "dlls/dwrite/scripts" );
  2559. dump_indic( "dlls/gdi32/uniscribe/indicsyllable.c" );
  2560. dump_vertical( "dlls/gdi32/vertical.c" );
  2561. dump_vertical( "dlls/wineps.drv/vertical.c" );
  2562. dump_intl_nls("nls/l_intl.nls");
  2563. dump_norm_table( "nls/normnfc.nls" );
  2564. dump_norm_table( "nls/normnfd.nls" );
  2565. dump_norm_table( "nls/normnfkc.nls" );
  2566. dump_norm_table( "nls/normnfkd.nls" );
  2567. dump_norm_table( "nls/normidna.nls" );
  2568. dump_sortkey_table( "nls/sortdefault.nls", "Windows 10 Sorting Weight Table.txt" );
  2569. foreach my $file (@allfiles) { dump_msdata_codepage( $file ); }
  2570. dump_eucjp_codepage();
  2571. dump_registry_script( "dlls/kernelbase/kernelbase.rgs", %registry_keys );
  2572. exit 0;
  2573. # Local Variables:
  2574. # compile-command: "./make_unicode"
  2575. # End: