//******************************************************************************* // // Copyright (C) 2003, International Business Machines // Corporation and others. All Rights Reserved. // // file name: conversion.txt // encoding: US-ASCII // tab size: 8 (not used) // indentation:4 // // created on: 2003jul15 // created by: Markus W. Scherer // // ICU resource bundle source file with test data for data-driven conversion tests. // //******************************************************************************* conversion { Info { Description { "Test data for conversion" } LongDescription { "Test data for data-driven conversion tests in icu/source/test/intltest/convtest.cpp\n" "Run intltest conversion\n" "Charset names starting with '*' are for testdata names.\n" "ICU callbacks are specified as strings with pairs of characters, each optional.\n" "Callback function - '?'=Sub '0'=Skip '.'=Stop '&'=Escape\n" "Callback option - a letter is passed in directly as const char * see ucnv_err.h\n" "Empty string: Sub callback with NULL option\n" "In order to specify a charset substitution character,\n" "add a NUL (U+0000) to the callback string followed by the subchar bytes as Latin-1\n" "characters. For example, for a Sub callback with no option and a subchar of FC FC,\n" "use the string \"?\x00\xFC\xFC\"\n" "fallbacks: per-direction boolean, currently only for fromUnicode; see Jitterbug 2401\n" "errorCode: (empty)==zero | invalid | illegal | truncated\n" } } TestData { toUnicode { Headers { "charset", "bytes", "unicode", "offsets", "flush", "fallbacks", "errorCode", "callback", "invalidChars" } Cases { { "UTF-16LE", :bin{ 310000d801dc00d902dc320000d8330001dc3400 }, "1\U00010001\U000500022\ufffd3\ufffd4", :intvector{ 0, 2, 2, 6, 6, 10, 12, 14, 16, 18 }, :int{1}, :int{0}, "", "?", :bin{""} } { "UTF-16LE", :bin{ 00 }, "", :intvector{}, :int{1}, :int{0}, "truncated", ".", :bin{ 00 } } { "UTF-16LE", :bin{ 00d800 }, "", :intvector{}, :int{1}, :int{0}, "truncated", ".", :bin{ 00d800 } } { "UTF-16BE", :bin{ 0031d800dc01d900dc020032d8000033dc010034 }, "1\U00010001\U000500022\ufffd3\ufffd4", :intvector{ 0, 2, 2, 6, 6, 10, 12, 14, 16, 18 }, :int{1}, :int{0}, "", "?", :bin{""} } { "UTF-16BE", :bin{ 00 }, "", :intvector{}, :int{1}, :int{0}, "truncated", ".", :bin{ 00 } } { "UTF-16BE", :bin{ d800dc }, "", :intvector{}, :int{1}, :int{0}, "truncated", ".", :bin{ d800dc } } // e4b8 is a partial sequence { "UTF-8", :bin{ 31e4ba8ce4b8 }, "1\u4e8c", :intvector{ 0, 1 }, :int{1}, :int{0}, "truncated", ".", :bin{ e4b8 } } { "UTF-8", :bin{ 31e4ba8ce4b8 }, "1\u4e8c\ufffd", :intvector{ 0, 1, 4 }, :int{1}, :int{0}, "", "?", :bin{""} } // LMBCS with escape callback (1292a0 is unassigned) { "LMBCS", :bin{ 12c9501292a01292a1 }, "\u4e2e%X12%X92%XA0\ue5c4", :intvector{ 0, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 6 }, :int{1}, :int{0}, "", "&", :bin{""} } // IMAP-mailbox-name with SUB // a a&AB~ a&AB\x0c a&AB- a&AB. a&. { "IMAP-mailbox-name", :bin{ 617f612641427e612641420c612641422d612641422e61262e }, "a\ufffda\ufffda\ufffda\ufffda\ufffda\ufffd", :intvector{ 0, 1, 2, 4, 7, 9, 12, 14, 17, 19, 22, 23 }, :int{1}, :int{0}, "", "?", :bin{""} } // using testdata_test1.cnv { "*test1", :bin{ 000506070809 }, "\u20ac\x05\x06\U00101234\ufffd\ufffd", :intvector{ 0, 1, 2, 3, 3, 4, 5 }, :int{1}, :int{0}, "", "", :bin{""} } // surrogates in CESU-8 { "CESU-8", :bin{ eda080eda081edb081 }, "\ud800\U00010401", :intvector{ 0, 3, 6 }, :int{1}, :int{0}, "", "", :bin{""} } // e080 is a partial sequence { "UTF-8", :bin{ 31ffe4ba8ce08061 }, "1\ufffd\u4e8c\ufffda", :intvector{ 0, 1, 2, 5, 7 }, :int{0}, :int{0}, "", "", :bin{ e080 } } // fbbfbfbfbf exceedes U+10ffff { "UTF-8", :bin{ 31fbbfbfbfbf61 }, "1\ufffda", :intvector{ 0, 1, 6 }, :int{0}, :int{0}, "", "", :bin{ fbbfbfbfbf } } // lead byte a2 without trail byte { "ibm-1363", :bin{ a2aea2 }, "\u00a1", :intvector{ 0 }, :int{1}, :int{0}, "truncated", ".", :bin{ a2 } } { "ibm-1363", :bin{ a2aea2 }, "\u00a1\u001a", :intvector{ 0, 2 }, :int{1}, :int{0}, "", "?", :bin{""} } // simple sample, no error handling { "UTF-8", :bin{ 61F48FBFBF }, "a\U0010FFFF", :intvector{ 0, 1, 1 }, :int{1}, :int{0}, "", "", :bin{""} } } } // --------------------------------------------------------------------- *** fromUnicode { Headers { "charset", "unicode", "bytes", "offsets", "flush", "fallbacks", "errorCode", "callback", "invalidUChars" } Cases { { "UTF-16LE", "1\U00010001\U000500022\ud8003\udc014", :bin{ 310000d801dc00d902dc3200fdff3300fdff3400 }, :intvector{ 0, 0, 1, 1, 1, 1, 3, 3, 3, 3, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9 }, :int{1}, :int{0}, "", "?", "" } { "UTF-16LE", "\ud800", :bin{""}, :intvector{}, :int{1}, :int{0}, "truncated", ".", "\ud800" } { "UTF-16BE", "1\U00010001\U000500022\ud8003\udc014", :bin{ 0031d800dc01d900dc020032fffd0033fffd0034 }, :intvector{ 0, 0, 1, 1, 1, 1, 3, 3, 3, 3, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9 }, :int{1}, :int{0}, "", "?", "" } { "UTF-16BE", "\ud800", :bin{""}, :intvector{}, :int{1}, :int{0}, "truncated", ".", "\ud800" } // escape callback { "ISCII", "A\u0901\U00023456\u0902B\U00023456C", :bin{ 41ef42a1255544383444255544433536a24225554438344425554443353643 }, :intvector{ 0, 1,1,1, 2,2,2,2,2,2, 2,2,2,2,2,2, 4, 5, 6,6,6,6,6,6, 6,6,6,6,6,6, 8 }, :int{1}, :int{0}, "", "&", "" } // escape callback (hex) { "iso-2022-jp", "\u3000\U00023456\u3001\U00023456B\u901c", :bin{ 1b244221211b284226237832333435363b1b244221221b284226237832333435363b42262378393031433b }, :intvector{ 0,0,0,0,0, 1,1,1,1,1,1,1,1,1,1,1,1, 3,3,3,3,3, 4,4,4,4,4,4,4,4,4,4,4,4, 6, 7,7,7,7,7,7,7,7 }, :int{1}, :int{0}, "", "&X", "" } // sub callback { "gb18030", "$\x7f\x80\u01f9\u20ac\u4e00\u9fa6\uffff\U00010000\U0010ffff", :bin{ 247f81308130a8bfa2e3d2bb82358f338431a43990308130e3329a35 }, :intvector{ 0, 1, 2, 2, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 6, 6, 7, 7, 7, 7, 8, 8, 8, 8, 10, 10, 10, 10 }, :int{1}, :int{0}, "", "?", "" } // skip callback { "ibm-930", "\u6D63\u6D64\u6D65\u6D66", :bin{ 0e5d5f5d63466b0f }, :intvector{ 0, 0, 0, 1, 1, 3, 3, 3 }, :int{1}, :int{0}, "", "0", "" } { "ibm-930", "\u6D63\u6D64\ud89a\u6D66", :bin{ 0e5d5f5d63466b0f }, :intvector{ 0, 0, 0, 1, 1, 3, 3, 3 }, :int{1}, :int{0}, "", "0", "" } { "ibm-930", "\u6D63\u6D64\ud89a\u6D66", :bin{ 0e5d5f5d63 }, :intvector{ 0, 0, 0, 1, 1 }, :int{1}, :int{0}, "illegal", "0i", "\ud89a" } // sub callback for supplementary code point { "LATIN1", "1\U000104012", :bin{ 311a32 }, :intvector{ 0, 1, 3 }, :int{1}, :int{0}, "", "", "" } { "ibm-920", "1\U000104012", :bin{ 311a32 }, :intvector{ 0, 1, 3 }, :int{1}, :int{0}, "", "", "" } // sub callback with AA as subchar { "ibm-920", "1\U000104012", :bin{ 31AA32 }, :intvector{ 0, 1, 3 }, :int{1}, :int{0}, "", "?\x00\xAA", "" } // same but not flushing { "LATIN1", "1\U000104012", :bin{ 311a32 }, :intvector{ 0, 1, 3 }, :int{0}, :int{0}, "", "", "\U00010401" } { "ibm-920", "1\U000104012", :bin{ 311a32 }, :intvector{ 0, 1, 3 }, :int{0}, :int{0}, "", "", "\U00010401" } // simple sample, no error handling { "UTF-8", "a\U0010FFFF", :bin{ 61F48FBFBF }, :intvector{ 0, 1, 1, 1, 1 }, :int{1}, :int{0}, "", "", "" } } } } }