From 4f2138ecfbdc58e5cb5b0c7d762197ef69752957 Mon Sep 17 00:00:00 2001
From: Simon Hausmann <simon.hausmann@nokia.com>
Date: Mon, 16 May 2011 14:04:31 +0200
Subject: [PATCH] Fix inconsistency between Qt and ICU in Shift-JIS codec with
 regards to ASCII range

Qt's Shift-JIS codec maps the characters 0x5c and 0x7e to unicode yen (0x5a)
and unicode overline (0x203e). ICU and (as it turns out) Symbian's native
Shift-JIS codec preserve 0x5c and 0x7e when converting to Unicode.

Qt's behaviour creates a problem when loading japanese web sites that are
encoded in Shift-JIS. When they reference external JavaScript files, those tend
to inherit the current page encoding (unless the character set is explicitly
specified). Consequently JavaScript tends to contain regular expressions (as a
built-in feature of the language), which in turn uses backslashes for escape
sequences. Therefore it is crucial that the encodings used to decode the script
preserve the ASCII range, i.e. do not convert 0x5c (ascii backslash) to
something else.

This patch corrects the behaviour of Qt's Shift-JIS codec to leave all
characters < 0x80 unaltered in the process of conversion to and from
Unicode.

Task: QTBUG-19335

Reviewed-by: Lars Knoll <lars.knoll@nokia.com>
(cherry picked from commit 8e321cd869da7ff1cf0168da41aa0246b44867cc)
---
 src/plugins/codecs/jp/qsjiscodec.cpp     |  4 +++-
 tests/auto/qtextcodec/tst_qtextcodec.cpp | 15 +++++++++++++++
 2 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/src/plugins/codecs/jp/qsjiscodec.cpp b/src/plugins/codecs/jp/qsjiscodec.cpp
index 49190d310f..3ea5300d51 100644
--- a/src/plugins/codecs/jp/qsjiscodec.cpp
+++ b/src/plugins/codecs/jp/qsjiscodec.cpp
@@ -155,7 +155,9 @@ QString QSjisCodec::convertToUnicode(const char* chars, int len, ConverterState
         uchar ch = chars[i];
         switch (nbuf) {
         case 0:
-            if (ch < 0x80 || IsKana(ch)) {
+            if (ch < 0x80) {
+                result += QValidChar(ch);
+            } else if (IsKana(ch)) {
                 // JIS X 0201 Latin or JIS X 0201 Kana
                 u = conv->jisx0201ToUnicode(ch);
                 result += QValidChar(u);
diff --git a/tests/auto/qtextcodec/tst_qtextcodec.cpp b/tests/auto/qtextcodec/tst_qtextcodec.cpp
index d122b0bde8..34d34fa90a 100644
--- a/tests/auto/qtextcodec/tst_qtextcodec.cpp
+++ b/tests/auto/qtextcodec/tst_qtextcodec.cpp
@@ -106,6 +106,8 @@ private slots:
 
     void moreToFromUnicode_data();
     void moreToFromUnicode();
+
+    void shiftJis();
 };
 
 void tst_QTextCodec::toUnicode_data()
@@ -2236,6 +2238,19 @@ void tst_QTextCodec::moreToFromUnicode()
     QCOMPARE(testData, cStr);
 }
 
+void tst_QTextCodec::shiftJis()
+{
+    QByteArray backslashTilde("\\~");
+    QTextCodec* codec = QTextCodec::codecForName("shift_jis");
+    QString string = codec->toUnicode(backslashTilde);
+    QCOMPARE(string.length(), 2);
+    QCOMPARE(string.at(0), QChar(QLatin1Char('\\')));
+    QCOMPARE(string.at(1), QChar(QLatin1Char('~')));
+
+    QByteArray encoded = codec->fromUnicode(string);
+    QCOMPARE(encoded, backslashTilde);
+}
+
 struct DontCrashAtExit {
     ~DontCrashAtExit() {
         QTextCodec *c = QTextCodec::codecForName("utf8");