Get rid of QTextCodec in QTextStream

Use QStringConverter instead. Also change the default
encoding of QTextStream to utf8.

Change-Id: I30682e75fe0462d1a937539f773640c83a2d82e1
Reviewed-by: Thiago Macieira <thiago.macieira@intel.com>
This commit is contained in:
Lars Knoll 2020-04-28 18:15:54 +02:00
parent 33bb695a28
commit 4d67b9dcfb
9 changed files with 114 additions and 509 deletions

View File

@ -62,7 +62,7 @@ static const int QTEXTSTREAM_BUFFERSIZE = 16384;
It's also common to use QTextStream to read console input and write
console output. QTextStream is locale aware, and will automatically decode
standard input using the correct codec. Example:
standard input using the correct encoding. Example:
\snippet code/src_corelib_io_qtextstream.cpp 1
@ -74,16 +74,16 @@ static const int QTEXTSTREAM_BUFFERSIZE = 16384;
buffer into the device and call flush() on the device.
Internally, QTextStream uses a Unicode based buffer, and
QTextCodec is used by QTextStream to automatically support
different character sets. By default, QTextCodec::codecForLocale()
is used for reading and writing, but you can also set the codec by
calling setCodec(). Automatic Unicode detection is also
QStringConverter is used by QTextStream to automatically support
different encodings. By default, UTF-8
is used for reading and writing, but you can also set the encoding by
calling setEncoding(). Automatic Unicode detection is also
supported. When this feature is enabled (the default behavior),
QTextStream will detect the UTF-16 or the UTF-32 BOM (Byte Order Mark) and
switch to the appropriate UTF codec when reading. QTextStream
QTextStream will detect the UTF-8, UTF-16 or the UTF-32 BOM (Byte Order Mark) and
switch to the appropriate UTF encoding when reading. QTextStream
does not write a BOM by default, but you can enable this by calling
setGenerateByteOrderMark(true). When QTextStream operates on a QString
directly, the codec is disabled.
directly, the encoding is disabled.
There are three general ways to use QTextStream when reading text
files:
@ -233,6 +233,7 @@ static const int QTEXTSTREAM_BUFFERSIZE = 16384;
#include <locale.h>
#include "private/qlocale_p.h"
#include "private/qstringconverter_p.h"
#include <stdlib.h>
#include <limits.h>
@ -325,12 +326,8 @@ QT_BEGIN_NAMESPACE
\internal
*/
QTextStreamPrivate::QTextStreamPrivate(QTextStream *q_ptr)
:
#if QT_CONFIG(textcodec)
readConverterSavedState(nullptr),
#endif
readConverterSavedStateOffset(0),
locale(QLocale::c())
: readConverterSavedStateOffset(0),
locale(QLocale::c())
{
this->q_ptr = q_ptr;
reset();
@ -347,34 +344,8 @@ QTextStreamPrivate::~QTextStreamPrivate()
#endif
delete device;
}
#if QT_CONFIG(textcodec)
delete readConverterSavedState;
#endif
}
#if QT_CONFIG(textcodec)
static void resetCodecConverterStateHelper(QTextCodec::ConverterState *state)
{
state->~State();
new (state) QTextCodec::ConverterState;
}
static void copyConverterStateHelper(QTextCodec::ConverterState *dest,
const QTextCodec::ConverterState *src)
{
// ### QTextCodec::ConverterState's copy constructors and assignments are
// private. This function copies the structure manually.
Q_ASSERT(!src->clearFn);
dest->flags = src->flags;
dest->remainingChars = src->remainingChars;
dest->invalidChars = src->invalidChars;
dest->state_data[0] = src->state_data[0];
dest->state_data[1] = src->state_data[1];
dest->state_data[2] = src->state_data[2];
dest->state_data[3] = src->state_data[3];
}
#endif
void QTextStreamPrivate::Params::reset()
{
realNumberPrecision = 6;
@ -403,15 +374,12 @@ void QTextStreamPrivate::reset()
readBufferStartDevicePos = 0;
lastTokenSize = 0;
#if QT_CONFIG(textcodec)
codec = QTextCodec::codecForLocale();
resetCodecConverterStateHelper(&readConverterState);
resetCodecConverterStateHelper(&writeConverterState);
delete readConverterSavedState;
readConverterSavedState = nullptr;
writeConverterState.flags |= QTextCodec::IgnoreHeader;
hasWrittenData = false;
generateBOM = false;
encoding = QStringConverter::Utf8;
toUtf16 = QStringDecoder(encoding);
fromUtf16 = QStringEncoder(encoding);
autoDetectUnicode = true;
#endif
}
/*!
@ -463,22 +431,19 @@ bool QTextStreamPrivate::fillReadBuffer(qint64 maxBytes)
if (bytesRead <= 0)
return false;
#if QT_CONFIG(textcodec)
// codec auto detection, explicitly defaults to locale encoding if the
// codec has been set to 0.
if (!codec || autoDetectUnicode) {
if (autoDetectUnicode) {
autoDetectUnicode = false;
codec = QTextCodec::codecForUtfText(QByteArray::fromRawData(buf, bytesRead), codec);
if (!codec) {
codec = QTextCodec::codecForLocale();
writeConverterState.flags |= QTextCodec::IgnoreHeader;
auto e = QStringConverter::encodingForData(buf, bytesRead);
// QStringConverter::Locale implies unknown, so keep the current encoding
if (e) {
encoding = *e;
toUtf16 = QStringDecoder(encoding);
fromUtf16 = QStringEncoder(encoding);
}
}
#if defined (QTEXTSTREAM_DEBUG)
qDebug("QTextStreamPrivate::fillReadBuffer(), using %s codec",
codec ? codec->name().constData() : "no");
#endif
qDebug("QTextStreamPrivate::fillReadBuffer(), using %s encoding", QStringConverter::nameForEncoding(encoding));
#endif
#if defined (QTEXTSTREAM_DEBUG)
@ -487,13 +452,7 @@ bool QTextStreamPrivate::fillReadBuffer(qint64 maxBytes)
#endif
int oldReadBufferSize = readBuffer.size();
#if QT_CONFIG(textcodec)
// convert to unicode
readBuffer += Q_LIKELY(codec) ? codec->toUnicode(buf, bytesRead, &readConverterState)
: QString::fromLatin1(buf, bytesRead);
#else
readBuffer += QString::fromLatin1(buf, bytesRead);
#endif
readBuffer += toUtf16(buf, bytesRead);
// remove all '\r\n' in the string.
if (readBuffer.size() > oldReadBufferSize && textModeEnabled) {
@ -569,23 +528,9 @@ void QTextStreamPrivate::flushWriteBuffer()
}
#endif
#if QT_CONFIG(textcodec)
if (!codec)
codec = QTextCodec::codecForLocale();
#if defined (QTEXTSTREAM_DEBUG)
qDebug("QTextStreamPrivate::flushWriteBuffer(), using %s codec (%s generating BOM)",
codec ? codec->name().constData() : "no",
!codec || (writeConverterState.flags & QTextCodec::IgnoreHeader) ? "not" : "");
#endif
// convert from unicode to raw data
// codec might be null if we're already inside global destructors (QTestCodec::codecForLocale returned null)
QByteArray data = Q_LIKELY(codec) ? codec->fromUnicode(writeBuffer.data(), writeBuffer.size(), &writeConverterState)
: writeBuffer.toLatin1();
#else
QByteArray data = writeBuffer.toLatin1();
#endif
QByteArray data = fromUtf16(writeBuffer);
writeBuffer.clear();
hasWrittenData = true;
// write raw data to the device
qint64 bytesWritten = device->write(data);
@ -788,18 +733,8 @@ inline void QTextStreamPrivate::consume(int size)
*/
inline void QTextStreamPrivate::saveConverterState(qint64 newPos)
{
#if QT_CONFIG(textcodec)
if (readConverterState.clearFn) {
// converter cannot be copied, so don't save anything
// don't update readBufferStartDevicePos either
return;
}
if (!readConverterSavedState)
readConverterSavedState = new QTextCodec::ConverterState;
copyConverterStateHelper(readConverterSavedState, &readConverterState);
#endif
// ### Hack, FIXME
memcpy((void *)&savedToUtf16, (void *)&toUtf16, sizeof(QStringDecoder));
readBufferStartDevicePos = newPos;
readConverterSavedStateOffset = 0;
}
@ -809,17 +744,11 @@ inline void QTextStreamPrivate::saveConverterState(qint64 newPos)
*/
inline void QTextStreamPrivate::restoreToSavedConverterState()
{
#if QT_CONFIG(textcodec)
if (readConverterSavedState) {
// we have a saved state
// that means the converter can be copied
copyConverterStateHelper(&readConverterState, readConverterSavedState);
} else {
// the only state we could save was the initial
// so reset to that
resetCodecConverterStateHelper(&readConverterState);
}
#endif
if (savedToUtf16.isValid())
memcpy((void *)&toUtf16, (void *)&savedToUtf16, sizeof(QStringDecoder));
else
toUtf16.resetState();
savedToUtf16 = QStringDecoder();
}
/*!
@ -1204,14 +1133,8 @@ bool QTextStream::seek(qint64 pos)
return false;
d->resetReadBuffer();
#if QT_CONFIG(textcodec)
// Reset the codec converter states.
resetCodecConverterStateHelper(&d->readConverterState);
resetCodecConverterStateHelper(&d->writeConverterState);
delete d->readConverterSavedState;
d->readConverterSavedState = nullptr;
d->writeConverterState.flags |= QTextCodec::IgnoreHeader;
#endif
d->toUtf16.resetState();
d->fromUtf16.resetState();
return true;
}
@ -1255,11 +1178,9 @@ qint64 QTextStream::pos() const
QTextStreamPrivate *thatd = const_cast<QTextStreamPrivate *>(d);
thatd->readBuffer.clear();
#if QT_CONFIG(textcodec)
thatd->restoreToSavedConverterState();
if (d->readBufferStartDevicePos == 0)
thatd->autoDetectUnicode = true;
#endif
// Rewind the device to get to the current position Ensure that
// readBufferOffset is unaffected by fillReadBuffer()
@ -1307,7 +1228,7 @@ void QTextStream::skipWhiteSpace()
replaced.
\note This function resets locale to the default locale ('C')
and codec to the default codec, QTextCodec::codecForLocale().
and encoding to the default encoding, UTF-8.
\sa device(), setString()
*/
@ -2579,10 +2500,9 @@ QTextStream &QTextStream::operator<<(double f)
/*!
Writes the string \a string to the stream, and returns a reference
to the QTextStream. The string is first encoded using the assigned
codec (the default codec is QTextCodec::codecForLocale()) before
it is written to the stream.
encoding (the default is UTF-8) before it is written to the stream.
\sa setFieldWidth(), setCodec()
\sa setFieldWidth(), setEncoding()
*/
QTextStream &QTextStream::operator<<(const QString &string)
{
@ -3134,7 +3054,6 @@ QTextStream &ws(QTextStream &stream)
Equivalent to QTextStream::setRealNumberPrecision(\a precision).
*/
#if QT_CONFIG(textcodec)
namespace Qt {
/*!
@ -3145,7 +3064,7 @@ namespace Qt {
/*!
Toggles insertion of the Byte Order Mark on \a stream when QTextStream is
used with a UTF codec.
used with a UTF encoding.
\sa QTextStream::setGenerateByteOrderMark(), {QTextStream manipulators}
*/
@ -3176,13 +3095,24 @@ QTextStream &bom(QTextStream &stream)
void QTextStream::setEncoding(QStringConverter::Encoding encoding)
{
Q_D(QTextStream);
if (d->encoding == encoding)
return;
qint64 seekPos = -1;
if (!d->readBuffer.isEmpty()) {
if (!d->device->isSequential()) {
seekPos = pos();
}
}
d->encoding = encoding;
#if QT_CONFIG(textcodec)
// FIXME: This is temporary until QTextStream is converted to use QStringConverter
const char *name = QStringConverter::nameForEncoding(encoding);
auto codec = QTextCodec::codecForName(name);
setCodec(codec);
#endif
d->toUtf16 = QStringDecoder(d->encoding);
bool generateBOM = d->hasWrittenData && d->generateBOM;
d->fromUtf16 = QStringEncoder(d->encoding,
generateBOM ? QStringEncoder::Flag::WriteBom : QStringEncoder::Flag::Default);
if (seekPos >=0 && !d->readBuffer.isEmpty())
seek(seekPos);
}
/*!
@ -3196,75 +3126,16 @@ QStringConverter::Encoding QTextStream::encoding() const
return d->encoding;
}
/*!
Sets the codec for this stream to \a codec. The codec is used for
decoding any data that is read from the assigned device, and for
encoding any data that is written. By default,
QTextCodec::codecForLocale() is used, and automatic unicode
detection is enabled.
If QTextStream operates on a string, this function does nothing.
\warning If you call this function while the text stream is reading
from an open sequential socket, the internal buffer may still contain
text decoded using the old codec.
\sa codec(), setAutoDetectUnicode(), setLocale()
*/
void QTextStream::setCodec(QTextCodec *codec)
{
Q_D(QTextStream);
qint64 seekPos = -1;
if (!d->readBuffer.isEmpty()) {
if (!d->device->isSequential()) {
seekPos = pos();
}
}
d->codec = codec;
if (seekPos >=0 && !d->readBuffer.isEmpty())
seek(seekPos);
}
/*!
Sets the codec for this stream to the QTextCodec for the encoding
specified by \a codecName. Common values for \c codecName include
"ISO 8859-1", "UTF-8", and "UTF-16". If the encoding isn't
recognized, nothing happens.
Example:
\snippet code/src_corelib_io_qtextstream.cpp 10
\sa QTextCodec::codecForName(), setLocale()
*/
void QTextStream::setCodec(const char *codecName)
{
QTextCodec *codec = QTextCodec::codecForName(codecName);
if (codec)
setCodec(codec);
}
/*!
Returns the codec that is current assigned to the stream.
\sa setCodec(), setAutoDetectUnicode(), locale()
*/
QTextCodec *QTextStream::codec() const
{
Q_D(const QTextStream);
return d->codec;
}
/*!
If \a enabled is true, QTextStream will attempt to detect Unicode encoding
by peeking into the stream data to see if it can find the UTF-8, UTF-16, or
UTF-32 Byte Order Mark (BOM). If this mark is found, QTextStream will
replace the current codec with the UTF codec.
replace the current encoding with the UTF encoding.
This function can be used together with setCodec(). It is common
to set the codec to UTF-8, and then enable UTF-16 detection.
This function can be used together with setEncoding(). It is common
to set the encoding to UTF-8, and then enable UTF-16 detection.
\sa autoDetectUnicode(), setCodec(), QTextCodec::codecForUtfText()
\sa autoDetectUnicode(), setEncoding()
*/
void QTextStream::setAutoDetectUnicode(bool enabled)
{
@ -3276,7 +3147,7 @@ void QTextStream::setAutoDetectUnicode(bool enabled)
Returns \c true if automatic Unicode detection is enabled, otherwise
returns \c false. Automatic Unicode detection is enabled by default.
\sa setAutoDetectUnicode(), setCodec(), QTextCodec::codecForUtfText()
\sa setAutoDetectUnicode(), setEncoding()
*/
bool QTextStream::autoDetectUnicode() const
{
@ -3285,7 +3156,7 @@ bool QTextStream::autoDetectUnicode() const
}
/*!
If \a generate is true and a UTF codec is used, QTextStream will insert
If \a generate is true and a UTF encoding is used, QTextStream will insert
the BOM (Byte Order Mark) before any data has been written to the
device. If \a generate is false, no BOM will be inserted. This function
must be called before any data is written. Otherwise, it does nothing.
@ -3295,14 +3166,16 @@ bool QTextStream::autoDetectUnicode() const
void QTextStream::setGenerateByteOrderMark(bool generate)
{
Q_D(QTextStream);
if (d->writeBuffer.isEmpty()) {
d->writeConverterState.flags.setFlag(QStringConverter::Flag::WriteBom, generate);
}
if (d->hasWrittenData || d->generateBOM == generate)
return;
d->generateBOM = generate;
d->fromUtf16 = QStringEncoder(d->encoding, generate ? QStringConverter::Flag::WriteBom : QStringConverter::Flag::Default);
}
/*!
Returns \c true if QTextStream is set to generate the UTF BOM (Byte Order
Mark) when using a UTF codec; otherwise returns \c false. UTF BOM generation is
Mark) when using a UTF encoding; otherwise returns \c false. UTF BOM generation is
set to false by default.
\sa setGenerateByteOrderMark()
@ -3310,11 +3183,9 @@ void QTextStream::setGenerateByteOrderMark(bool generate)
bool QTextStream::generateByteOrderMark() const
{
Q_D(const QTextStream);
return (d->writeConverterState.flags & QStringConverter::Flag::WriteBom);
return d->generateBOM;
}
#endif
/*!
\since 4.5
@ -3378,9 +3249,7 @@ QTextStream &reset(QTextStream &s) { return Qt::reset(s); }
QTextStream &ws(QTextStream &s) { return Qt::ws(s); }
#if QT_CONFIG(textcodec)
QTextStream &bom(QTextStream &s) { return Qt::bom(s); }
#endif
} // namespace QTextStreamFunctions
#endif
@ -3416,10 +3285,8 @@ Q_CORE_EXPORT QTextStream &reset(QTextStream &s) { return Qt::reset(s); }
Q_CORE_EXPORT QTextStream &ws(QTextStream &s) { return Qt::ws(s); }
#if QT_CONFIG(textcodec)
Q_CORE_EXPORT QTextStream &bom(QTextStream &s) { return Qt::bom(s); }
#endif
#endif
QT_END_NAMESPACE

View File

@ -101,15 +101,10 @@ public:
void setEncoding(QStringConverter::Encoding encoding);
QStringConverter::Encoding encoding() const;
#if QT_CONFIG(textcodec)
void setCodec(QTextCodec *codec);
void setCodec(const char *codecName);
QTextCodec *codec() const;
void setAutoDetectUnicode(bool enabled);
bool autoDetectUnicode() const;
void setGenerateByteOrderMark(bool generate);
bool generateByteOrderMark() const;
#endif
void setLocale(const QLocale &locale);
QLocale locale() const;

View File

@ -54,9 +54,6 @@
#include <QtCore/private/qglobal_p.h>
#include "qtextstream.h"
#if QT_CONFIG(textcodec)
#include "qtextcodec.h"
#endif
QT_BEGIN_NAMESPACE
@ -119,13 +116,9 @@ public:
QIODevice::OpenMode stringOpenMode;
QStringConverter::Encoding encoding = QStringConverter::Utf8;
#if QT_CONFIG(textcodec)
// codec
QTextCodec *codec;
QTextCodec::ConverterState readConverterState;
QTextCodec::ConverterState writeConverterState;
QTextCodec::ConverterState *readConverterSavedState;
#endif
QStringEncoder fromUtf16;
QStringDecoder toUtf16;
QStringDecoder savedToUtf16;
QString writeBuffer;
QString readBuffer;
@ -142,9 +135,9 @@ public:
int lastTokenSize;
bool deleteDevice;
#if QT_CONFIG(textcodec)
bool autoDetectUnicode;
#endif
bool hasWrittenData = false;
bool generateBOM = false;
// i/o
enum TokenDelimiter {

View File

@ -3648,17 +3648,10 @@ bool QDomAttrPrivate::specified() const
If \a encodeEOLs is true, characters will be escaped to survive End-of-Line Handling.
*/
static QString encodeText(const QString &str,
QTextStream &s,
const bool encodeQuotes = true,
const bool performAVN = false,
const bool encodeEOLs = false)
{
#if !QT_CONFIG(textcodec)
Q_UNUSED(s);
#else
const QTextCodec *const codec = s.codec();
Q_ASSERT(codec);
#endif
QString retval(str);
int len = retval.length();
int i = 0;
@ -3695,19 +3688,7 @@ static QString encodeText(const QString &str,
len += 4;
i += 5;
} else {
#if QT_CONFIG(textcodec)
if(codec->canEncode(ati))
++i;
else
#endif
{
// We have to use a character reference to get it through.
const ushort codepoint(ati.unicode());
const QString replacement(QLatin1String("&#x") + QString::number(codepoint, 16) + QLatin1Char(';'));
retval.replace(i, 1, replacement);
i += replacement.length();
len += replacement.length() - 1;
}
++i;
}
}
@ -3717,9 +3698,9 @@ static QString encodeText(const QString &str,
void QDomAttrPrivate::save(QTextStream& s, int, int) const
{
if (namespaceURI.isNull()) {
s << name << "=\"" << encodeText(value, s, true, true) << '\"';
s << name << "=\"" << encodeText(value, true, true) << '\"';
} else {
s << prefix << ':' << name << "=\"" << encodeText(value, s, true, true) << '\"';
s << prefix << ':' << name << "=\"" << encodeText(value, true, true) << '\"';
/* This is a fix for 138243, as good as it gets.
*
* QDomElementPrivate::save() output a namespace declaration if
@ -3733,7 +3714,7 @@ void QDomAttrPrivate::save(QTextStream& s, int, int) const
* arrive in those situations. */
if(!ownerNode ||
ownerNode->prefix != prefix) {
s << " xmlns:" << prefix << "=\"" << encodeText(namespaceURI, s, true, true) << '\"';
s << " xmlns:" << prefix << "=\"" << encodeText(namespaceURI, true, true) << '\"';
}
}
}
@ -4082,7 +4063,7 @@ void QDomElementPrivate::save(QTextStream& s, int depth, int indent) const
qName = prefix + QLatin1Char(':') + name;
nsDecl = QLatin1String(" xmlns:") + prefix;
}
nsDecl += QLatin1String("=\"") + encodeText(namespaceURI, s) + QLatin1Char('\"');
nsDecl += QLatin1String("=\"") + encodeText(namespaceURI) + QLatin1Char('\"');
}
s << '<' << qName << nsDecl;
@ -4094,9 +4075,9 @@ void QDomElementPrivate::save(QTextStream& s, int depth, int indent) const
for (; it != m_attr->map.constEnd(); ++it) {
s << ' ';
if (it.value()->namespaceURI.isNull()) {
s << it.value()->name << "=\"" << encodeText(it.value()->value, s, true, true) << '\"';
s << it.value()->name << "=\"" << encodeText(it.value()->value, true, true) << '\"';
} else {
s << it.value()->prefix << ':' << it.value()->name << "=\"" << encodeText(it.value()->value, s, true, true) << '\"';
s << it.value()->prefix << ':' << it.value()->name << "=\"" << encodeText(it.value()->value, true, true) << '\"';
/* This is a fix for 138243, as good as it gets.
*
* QDomElementPrivate::save() output a namespace declaration if
@ -4111,7 +4092,7 @@ void QDomElementPrivate::save(QTextStream& s, int depth, int indent) const
if((!it.value()->ownerNode ||
it.value()->ownerNode->prefix != it.value()->prefix) &&
!outputtedPrefixes.hasSeen(it.value()->prefix)) {
s << " xmlns:" << it.value()->prefix << "=\"" << encodeText(it.value()->namespaceURI, s, true, true) << '\"';
s << " xmlns:" << it.value()->prefix << "=\"" << encodeText(it.value()->namespaceURI, true, true) << '\"';
}
}
}
@ -4692,7 +4673,7 @@ QDomTextPrivate* QDomTextPrivate::splitText(int offset)
void QDomTextPrivate::save(QTextStream& s, int, int) const
{
QDomTextPrivate *that = const_cast<QDomTextPrivate*>(this);
s << encodeText(value, s, !(that->parent() && that->parent()->isElement()), false, true);
s << encodeText(value, !(that->parent() && that->parent()->isElement()), false, true);
}
/**************************************************************

View File

@ -30,7 +30,6 @@
#include <QtTest/QtTest>
#include <QPair>
#include <QTextCodec>
#include <QSysInfo>
#include <QLatin1String>
@ -50,8 +49,6 @@ private slots:
void qCoreAppStartupFunction();
void qCoreAppStartupFunctionRestart();
void integerForSize();
void qprintable();
void qprintable_data();
void buildAbiEndianness();
void testqOverload();
};
@ -450,62 +447,6 @@ void tst_QGlobal::integerForSize()
typedef QPair<const char *, const char *> stringpair;
Q_DECLARE_METATYPE(stringpair)
void tst_QGlobal::qprintable()
{
QFETCH(QVector<stringpair>, localestrings);
QFETCH(int, utf8index);
QVERIFY(utf8index >= 0 && utf8index < localestrings.count());
if (utf8index < 0 || utf8index >= localestrings.count())
return;
const char *const utf8string = localestrings.at(utf8index).second;
QString string = QString::fromUtf8(utf8string);
for (const stringpair &pair : qAsConst(localestrings)) {
QTextCodec *codec = QTextCodec::codecForName(pair.first);
if (!codec)
continue;
QTextCodec::setCodecForLocale(codec);
// test qPrintable()
QVERIFY(qstrcmp(qPrintable(string), pair.second) == 0);
for (const stringpair &pair2 : qAsConst(localestrings)) {
if (pair2.second == pair.second)
continue;
QVERIFY(qstrcmp(qPrintable(string), pair2.second) != 0);
}
// test qUtf8Printable()
QVERIFY(qstrcmp(qUtf8Printable(string), utf8string) == 0);
for (const stringpair &pair2 : qAsConst(localestrings)) {
if (qstrcmp(pair2.second, utf8string) == 0)
continue;
QVERIFY(qstrcmp(qUtf8Printable(string), pair2.second) != 0);
}
}
QTextCodec::setCodecForLocale(0);
}
void tst_QGlobal::qprintable_data()
{
QTest::addColumn<QVector<stringpair> >("localestrings");
QTest::addColumn<int>("utf8index"); // index of utf8 string
// Unicode: HIRAGANA LETTER A, I, U, E, O (U+3442, U+3444, U+3446, U+3448, U+344a)
static const char *const utf8string = "\xe3\x81\x82\xe3\x81\x84\xe3\x81\x86\xe3\x81\x88\xe3\x81\x8a";
static const char *const eucjpstring = "\xa4\xa2\xa4\xa4\xa4\xa6\xa4\xa8\xa4\xaa";
static const char *const sjisstring = "\x82\xa0\x82\xa2\x82\xa4\x82\xa6\x82\xa8";
QVector<stringpair> japanesestrings;
japanesestrings << stringpair("UTF-8", utf8string)
<< stringpair("EUC-JP", eucjpstring)
<< stringpair("Shift_JIS", sjisstring);
QTest::newRow("Japanese") << japanesestrings << 0;
}
void tst_QGlobal::buildAbiEndianness()
{
#if Q_BYTE_ORDER == Q_LITTLE_ENDIAN

View File

@ -37,10 +37,10 @@
#include <QDebug>
#include <QElapsedTimer>
#include <QFile>
#include <QStringConverter>
#include <QTcpSocket>
#include <QTemporaryDir>
#include <QTextStream>
#include <QTextCodec>
#if QT_CONFIG(process)
# include <QProcess>
#endif
@ -245,7 +245,7 @@ private:
void runOnExit()
{
QByteArray buffer;
QTextStream(&buffer) << "This will try to use QTextCodec::codecForLocale" << Qt::endl;
QTextStream(&buffer) << "This will try to use QStringConverter::Utf8" << Qt::endl;
}
Q_DESTRUCTOR_FUNCTION(runOnExit)
@ -277,8 +277,8 @@ void tst_QTextStream::getSetCheck()
{
// Initialize codecs
QTextStream obj1;
// QTextCodec * QTextStream::encoding()
// void QTextStream::setEncoding()
// QTextStream::encoding()
// QTextStream::setEncoding()
obj1.setEncoding(QStringConverter::Utf32BE);
QCOMPARE(QStringConverter::Utf32BE, obj1.encoding());
obj1.setEncoding(QStringConverter::Utf8);
@ -402,7 +402,7 @@ void tst_QTextStream::cleanupTestCase()
void tst_QTextStream::construction()
{
QTextStream stream;
QCOMPARE(stream.codec(), QTextCodec::codecForLocale());
QCOMPARE(stream.encoding(), QStringConverter::Utf8);
QCOMPARE(stream.device(), static_cast<QIODevice *>(0));
QCOMPARE(stream.string(), static_cast<QString *>(0));
@ -1766,7 +1766,6 @@ void tst_QTextStream::utf8IncompleteAtBufferBoundary()
QFile::remove(testFileName);
QFile data(testFileName);
QTextCodec *utf8Codec = QTextCodec::codecForMib(106);
QString lineContents = QString::fromUtf8("\342\200\223" // U+2013 EN DASH
"\342\200\223"
"\342\200\223"

View File

@ -1,78 +0,0 @@
<?xml version="1.0" encoding="euc-jp"?>
<!DOCTYPE 週報 SYSTEM "weekly-euc-jp.dtd">
<!-- 週報サンプル -->
<週報>
<年月週>
<年度>1997</年度>
<月度>1</月度>
<週>1</週>
</年月週>
<氏名>
<氏>山田</氏>
<名>太郎</名>
</氏名>
<業務報告リスト>
<業務報告>
<業務名>XMLエディターの作成</業務名>
<業務コード>X3355-23</業務コード>
<工数管理>
<見積もり工数>1600</見積もり工数>
<実績工数>320</実績工数>
<当月見積もり工数>160</当月見積もり工数>
<当月実績工数>24</当月実績工数>
</工数管理>
<予定項目リスト>
<予定項目>
<P>XMLエディターの基本仕様の作成</P>
</予定項目>
</予定項目リスト>
<実施事項リスト>
<実施事項>
<P>XMLエディターの基本仕様の作成</P>
</実施事項>
<実施事項>
<P>競合他社製品の機能調査</P>
</実施事項>
</実施事項リスト>
<上長への要請事項リスト>
<上長への要請事項>
<P>特になし</P>
</上長への要請事項>
</上長への要請事項リスト>
<問題点対策>
<P>XMLとは何かわからない。</P>
</問題点対策>
</業務報告>
<業務報告>
<業務名>検索エンジンの開発</業務名>
<業務コード>S8821-76</業務コード>
<工数管理>
<見積もり工数>120</見積もり工数>
<実績工数>6</実績工数>
<当月見積もり工数>32</当月見積もり工数>
<当月実績工数>2</当月実績工数>
</工数管理>
<予定項目リスト>
<予定項目>
<P><A href="http://www.goo.ne.jp">goo</A>の機能を調べてみる</P>
</予定項目>
</予定項目リスト>
<実施事項リスト>
<実施事項>
<P>更に、どういう検索エンジンがあるか調査する</P>
</実施事項>
</実施事項リスト>
<上長への要請事項リスト>
<上長への要請事項>
<P>開発をするのはめんどうなので、Yahoo!を買収して下さい。</P>
</上長への要請事項>
</上長への要請事項リスト>
<問題点対策>
<P>検索エンジンで車を走らせることができない。(要調査)</P>
</問題点対策>
</業務報告>
</業務報告リスト>
</週報>

View File

@ -1,78 +0,0 @@
<?xml version="1.0" encoding="iso-2022-jp"?>
<!DOCTYPE $B=5Js(B SYSTEM "weekly-iso-2022-jp.dtd">
<!-- $B=5Js%5%s%W%k(B -->
<$B=5Js(B>
<$BG/7n=5(B>
<$BG/EY(B>1997</$BG/EY(B>
<$B7nEY(B>1</$B7nEY(B>
<$B=5(B>1</$B=5(B>
</$BG/7n=5(B>
<$B;aL>(B>
<$B;a(B>$B;3ED(B</$B;a(B>
<$BL>(B>$BB@O:(B</$BL>(B>
</$B;aL>(B>
<$B6HL3Js9p%j%9%H(B>
<$B6HL3Js9p(B>
<$B6HL3L>(B>XML$B%(%G%#%?!<$N:n@.(B</$B6HL3L>(B>
<$B6HL3%3!<%I(B>X3355-23</$B6HL3%3!<%I(B>
<$B9)?t4IM}(B>
<$B8+@Q$b$j9)?t(B>1600</$B8+@Q$b$j9)?t(B>
<$B<B@S9)?t(B>320</$B<B@S9)?t(B>
<$BEv7n8+@Q$b$j9)?t(B>160</$BEv7n8+@Q$b$j9)?t(B>
<$BEv7n<B@S9)?t(B>24</$BEv7n<B@S9)?t(B>
</$B9)?t4IM}(B>
<$BM=Dj9`L\%j%9%H(B>
<$BM=Dj9`L\(B>
<P>XML$B%(%G%#%?!<$N4pK\;EMM$N:n@.(B</P>
</$BM=Dj9`L\(B>
</$BM=Dj9`L\%j%9%H(B>
<$B<B;\;v9`%j%9%H(B>
<$B<B;\;v9`(B>
<P>XML$B%(%G%#%?!<$N4pK\;EMM$N:n@.(B</P>
</$B<B;\;v9`(B>
<$B<B;\;v9`(B>
<P>$B6%9gB><R@=IJ$N5!G=D4::(B</P>
</$B<B;\;v9`(B>
</$B<B;\;v9`%j%9%H(B>
<$B>eD9$X$NMW@A;v9`%j%9%H(B>
<$B>eD9$X$NMW@A;v9`(B>
<P>$BFC$K$J$7(B</P>
</$B>eD9$X$NMW@A;v9`(B>
</$B>eD9$X$NMW@A;v9`%j%9%H(B>
<$BLdBjE@BP:v(B>
<P>XML$B$H$O2?$+$o$+$i$J$$!#(B</P>
</$BLdBjE@BP:v(B>
</$B6HL3Js9p(B>
<$B6HL3Js9p(B>
<$B6HL3L>(B>$B8!:w%(%s%8%s$N3+H/(B</$B6HL3L>(B>
<$B6HL3%3!<%I(B>S8821-76</$B6HL3%3!<%I(B>
<$B9)?t4IM}(B>
<$B8+@Q$b$j9)?t(B>120</$B8+@Q$b$j9)?t(B>
<$B<B@S9)?t(B>6</$B<B@S9)?t(B>
<$BEv7n8+@Q$b$j9)?t(B>32</$BEv7n8+@Q$b$j9)?t(B>
<$BEv7n<B@S9)?t(B>2</$BEv7n<B@S9)?t(B>
</$B9)?t4IM}(B>
<$BM=Dj9`L\%j%9%H(B>
<$BM=Dj9`L\(B>
<P><A href="http://www.goo.ne.jp">goo</A>$B$N5!G=$rD4$Y$F$_$k(B</P>
</$BM=Dj9`L\(B>
</$BM=Dj9`L\%j%9%H(B>
<$B<B;\;v9`%j%9%H(B>
<$B<B;\;v9`(B>
<P>$B99$K!"$I$&$$$&8!:w%(%s%8%s$,$"$k$+D4::$9$k(B</P>
</$B<B;\;v9`(B>
</$B<B;\;v9`%j%9%H(B>
<$B>eD9$X$NMW@A;v9`%j%9%H(B>
<$B>eD9$X$NMW@A;v9`(B>
<P>$B3+H/$r$9$k$N$O$a$s$I$&$J$N$G!"(BYahoo!$B$rGc<}$7$F2<$5$$!#(B</P>
</$B>eD9$X$NMW@A;v9`(B>
</$B>eD9$X$NMW@A;v9`%j%9%H(B>
<$BLdBjE@BP:v(B>
<P>$B8!:w%(%s%8%s$G<V$rAv$i$;$k$3$H$,$G$-$J$$!#!JMWD4::!K(B</P>
</$BLdBjE@BP:v(B>
</$B6HL3Js9p(B>
</$B6HL3Js9p%j%9%H(B>
</$B=5Js(B>

View File

@ -299,8 +299,6 @@ void tst_QDom::toString_01_data()
QTest::newRow( "04" ) << QString(prefix + "/doc04.xml");
QTest::newRow( "05" ) << QString(prefix + "/doc05.xml");
QTest::newRow( "euc-jp" ) << QString(prefix + "/doc_euc-jp.xml");
QTest::newRow( "iso-2022-jp" ) << QString(prefix + "/doc_iso-2022-jp.xml");
QTest::newRow( "little-endian" ) << QString(prefix + "/doc_little-endian.xml");
QTest::newRow( "utf-16" ) << QString(prefix + "/doc_utf-16.xml");
QTest::newRow( "utf-8" ) << QString(prefix + "/doc_utf-8.xml");
@ -576,48 +574,39 @@ void tst_QDom::saveWithSerialization() const
// Read the document
QVERIFY(doc.setContent(&f));
QByteArray codecName;
foreach (codecName, m_testCodecs) {
QByteArray storage;
QBuffer writeDevice(&storage);
QVERIFY(writeDevice.open(QIODevice::WriteOnly));
/* Write out doc in the specified codec. */
QByteArray storage;
QBuffer writeDevice(&storage);
QVERIFY(writeDevice.open(QIODevice::WriteOnly));
QTextStream s(&writeDevice);
QTextStream s(&writeDevice);
QTextCodec *codec = QTextCodec::codecForName(codecName);
QVERIFY2(codec, qPrintable(QString::fromLatin1("Failed to load codec %1").arg(QString::fromLatin1(codecName.constData()))));
s.setCodec(codec);
doc.save(s, 0, QDomNode::EncodingFromTextStream);
s.flush();
writeDevice.close();
doc.save(s, 0, QDomNode::EncodingFromTextStream);
s.flush();
writeDevice.close();
QBuffer readDevice(&storage);
QVERIFY(readDevice.open(QIODevice::ReadOnly));
QBuffer readDevice(&storage);
QVERIFY(readDevice.open(QIODevice::ReadOnly));
QDomDocument result;
QDomDocument result;
QString msg;
int line = 0;
int column = 0;
QString msg;
int line = 0;
int column = 0;
QVERIFY2(result.setContent(&readDevice, &msg, &line, &column),
qPrintable(QString::fromLatin1("Failed: line %2, column %3: %4, content: %5")
.arg(QString::number(line),
QString::number(column),
msg,
QString::fromUtf8(storage))));
if (!compareDocuments(doc, result))
{
QCOMPARE(doc.toString(), result.toString());
QVERIFY2(result.setContent(&readDevice, &msg, &line, &column),
qPrintable(QString::fromLatin1("Failed for codec %1: line %2, column %3: %4, content: %5")
.arg(QString::fromLatin1(codecName.constData()),
QString::number(line),
QString::number(column),
msg,
codec->toUnicode(storage))));
if(!compareDocuments(doc, result))
{
QCOMPARE(doc.toString(), result.toString());
/* We put this one here as well, in case the QCOMPARE above for some strange reason
* nevertheless succeeds. */
QVERIFY2(false, qPrintable(QString::fromLatin1("Failed for codec %1").arg(QString::fromLatin1(codecName.constData()))));
}
/* We put this one here as well, in case the QCOMPARE above for some strange reason
* nevertheless succeeds. */
QVERIFY2(false, qPrintable(QString::fromLatin1("Failed to serialize test data")));
}
}
@ -633,8 +622,6 @@ void tst_QDom::saveWithSerialization_data() const
QTest::newRow("doc04.xml") << QString(prefix + "/doc04.xml");
QTest::newRow("doc05.xml") << QString(prefix + "/doc05.xml");
QTest::newRow("doc_euc-jp.xml") << QString(prefix + "/doc_euc-jp.xml");
QTest::newRow("doc_iso-2022-jp.xml") << QString(prefix + "/doc_iso-2022-jp.xml");
QTest::newRow("doc_little-endian.xml") << QString(prefix + "/doc_little-endian.xml");
QTest::newRow("doc_utf-16.xml") << QString(prefix + "/doc_utf-16.xml");
QTest::newRow("doc_utf-8.xml") << QString(prefix + "/doc_utf-8.xml");
@ -1800,7 +1787,6 @@ void tst_QDom::germanUmlautToByteArray() const
QBuffer buffer(&data);
QVERIFY(buffer.open(QIODevice::WriteOnly));
QTextStream ts(&buffer);
ts.setCodec("UTF-8");
ts << d.toString();
buffer.close();
@ -1832,7 +1818,6 @@ void tst_QDom::germanUmlautToFile() const
QTemporaryFile file;
QVERIFY(file.open());
QTextStream ts(&file);
ts.setCodec("UTF-8");
ts << d.toString();
file.close();