AuroraMiddleware/qt5base-lts
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
973e74399e
qt5base-lts/tests/auto/testlib/selftests/expected_cmptest.junitxml
Tor Arne Vestbø 973e74399e testlib: Improve JUnit XML conformance 
The JUnit test framework did not initially have any XML reporting
facilities built in. Instead, the XML report was generated by the
Apache Ant JUnit task:

 https://github.com/apache/ant/search?q=filename%3AXMLJUnitResultFormatter.java

Many users interacted with these reports via the Jenkins JUnit plugin,
which provided graphical visualization of the test results:

 https://plugins.jenkins.io/junit/

Due to the lack of an official XML schema for the Apache Ant JUnit
report there was some confusion about what the actual format was.
People started documenting the de-facto format, both as produced
by Ant, and as consumed by Jenkins:

 https://github.com/windyroad/JUnit-Schema/blob/master/JUnit.xsd
 https://github.com/junit-team/junit5/search?q=filename%3Ajenkins-junit.xsd

The XML produced by the Qt Test JUnit reporter was far from these
schemas, causing issues when importing results into tools such
as Jenkins, Allure2, or Test Center.

The following changes have been made to improve conformance:

  - The 'timestamp' attribute on <testsuite> is is now in ISO
    8601 local time, without any time zone specified
  - The 'hostname' attribute on <testsuite> is now included
  - The 'classname' attribute on <testcase> is now included
  - The non-standard 'result' attribute on <testcase> has
    been removed
  - The non-standard 'result' attribute on <failure> has
    been renamed to 'type'
  - The <system-out> element on <testsuite> is always included,
    even when empty
  - The non-standard 'tag' attribute on <failure> has been
    removed. Data-driven tests are now represented as individual
    <testcase> elements, e.g.:

      <testcase name="someTest(someData X)" ...>
      <testcase name="someTest(someData Y)" ...>
      <testcase name="someTest(someData Z)" ...>

The resulting XML validates against both the de-facto Apache Ant
'JUnit 4' schema and the Jenkins JUnit plugin schema.

Task-number: QTBUG-95424
Change-Id: I6fc9abedbfb319f2545b99b37d059b18c16776ff
Reviewed-by: Mårten Nordheim <marten.nordheim@qt.io>
2021-08-05 03:58:49 +02:00

250 lines
14 KiB
XML
Raw Blame History

<?xml version="1.0" encoding="UTF-8" ?>
<testsuite name="tst_Cmptest" timestamp="@TEST_START_TIME@" hostname="@HOSTNAME@" tests="67" failures="46" errors="0" time="@TEST_DURATION@">
<properties>
<property name="QTestVersion" value="@INSERT_QT_VERSION_HERE@"/>
<property name="QtVersion" value="@INSERT_QT_VERSION_HERE@"/>
<property name="QtBuild" value=""/>
</properties>
<testcase name="initTestCase" classname="tst_Cmptest" time="@TEST_DURATION@"/>
<testcase name="compare_unregistered_enums" classname="tst_Cmptest" time="@TEST_DURATION@">
<failure type="fail" message="Compared values are not the same
Actual (MyUnregisteredEnumValue1): 0
Expected (MyUnregisteredEnumValue2): 1"/>
</testcase>
<testcase name="compare_registered_enums" classname="tst_Cmptest" time="@TEST_DURATION@">
<failure type="fail" message="Compared values are not the same
Actual (Qt::Monday): Monday
Expected (Qt::Sunday): Sunday"/>
</testcase>
<testcase name="compare_class_enums" classname="tst_Cmptest" time="@TEST_DURATION@">
<failure type="fail" message="Compared values are not the same
Actual (MyClassEnum::MyClassEnumValue1): MyClassEnumValue1
Expected (MyClassEnum::MyClassEnumValue2): MyClassEnumValue2"/>
</testcase>
<testcase name="test_windowflags(pass)" classname="tst_Cmptest" time="@TEST_DURATION@"/>
<testcase name="test_windowflags(fail1)" classname="tst_Cmptest" time="@TEST_DURATION@">
<failure type="fail" message="Compared values are not the same
Actual (actualWindowFlags) : Window|WindowSystemMenuHint|WindowStaysOnBottomHint
Expected (expectedWindowFlags): Window|FramelessWindowHint|WindowSystemMenuHint|WindowStaysOnBottomHint"/>
</testcase>
<testcase name="test_windowflags(fail2)" classname="tst_Cmptest" time="@TEST_DURATION@">
<failure type="fail" message="Compared values are not the same
Actual (actualWindowFlags) : Window
Expected (expectedWindowFlags): Window|FramelessWindowHint"/>
</testcase>
<testcase name="test_unregistered_flags(pass)" classname="tst_Cmptest" time="@TEST_DURATION@"/>
<testcase name="test_unregistered_flags(fail1)" classname="tst_Cmptest" time="@TEST_DURATION@">
<failure type="fail" message="Compared values are not the same
Actual (actualFlags) : 0x3
Expected (expectedFlags): 0x5"/>
</testcase>
<testcase name="test_unregistered_flags(fail2)" classname="tst_Cmptest" time="@TEST_DURATION@">
<failure type="fail" message="Compared values are not the same
Actual (actualFlags) : 0x1
Expected (expectedFlags): 0x5"/>
</testcase>
<testcase name="compare_boolfuncs" classname="tst_Cmptest" time="@TEST_DURATION@"/>
<testcase name="compare_to_nullptr" classname="tst_Cmptest" time="@TEST_DURATION@"/>
<testcase name="compare_pointerfuncs" classname="tst_Cmptest" time="@TEST_DURATION@"/>
<testcase name="compare_tostring(int, string)" classname="tst_Cmptest" time="@TEST_DURATION@">
<failure type="fail" message="Compared values are not the same
Actual (actual) : QVariant(int,123)
Expected (expected): QVariant(QString,hi)"/>
</testcase>
<testcase name="compare_tostring(both invalid)" classname="tst_Cmptest" time="@TEST_DURATION@"/>
<testcase name="compare_tostring(null hash, invalid)" classname="tst_Cmptest" time="@TEST_DURATION@">
<failure type="fail" message="Compared values are not the same
Actual (actual) : QVariant(QVariantHash)
Expected (expected): QVariant()"/>
</testcase>
<testcase name="compare_tostring(string, null user type)" classname="tst_Cmptest" time="@TEST_DURATION@">
<failure type="fail" message="Compared values are not the same
Actual (actual) : QVariant(QString,A simple string)
Expected (expected): QVariant(PhonyClass)"/>
</testcase>
<testcase name="compare_tostring(both non&#x002D;null user type)" classname="tst_Cmptest" time="@TEST_DURATION@">
<failure type="fail" message="Compared values are not the same
Actual (actual) : QVariant(PhonyClass,&lt;value not representable as string&gt;)
Expected (expected): QVariant(PhonyClass,&lt;value not representable as string&gt;)"/>
</testcase>
<testcase name="compareQObjects" classname="tst_Cmptest" time="@TEST_DURATION@">
<failure type="fail" message="Compared QObject pointers are not the same
Actual (&amp;object1): QObject/&quot;object1&quot;
Expected (&amp;object2): QObject/&quot;object2&quot;"/>
</testcase>
<testcase name="compareQStringLists(empty lists)" classname="tst_Cmptest" time="@TEST_DURATION@"/>
<testcase name="compareQStringLists(equal lists)" classname="tst_Cmptest" time="@TEST_DURATION@"/>
<testcase name="compareQStringLists(last item different)" classname="tst_Cmptest" time="@TEST_DURATION@">
<failure type="fail" message="Compared lists differ at index 2.
Actual (opA): &quot;string3&quot;
Expected (opB): &quot;DIFFERS&quot;"/>
</testcase>
<testcase name="compareQStringLists(second&#x002D;last item different)" classname="tst_Cmptest" time="@TEST_DURATION@">
<failure type="fail" message="Compared lists differ at index 2.
Actual (opA): &quot;string3&quot;
Expected (opB): &quot;DIFFERS&quot;"/>
</testcase>
<testcase name="compareQStringLists(prefix)" classname="tst_Cmptest" time="@TEST_DURATION@">
<failure type="fail" message="Compared lists have different sizes.
Actual (opA) size: 2
Expected (opB) size: 1"/>
</testcase>
<testcase name="compareQStringLists(short list second)" classname="tst_Cmptest" time="@TEST_DURATION@">
<failure type="fail" message="Compared lists have different sizes.
Actual (opA) size: 12
Expected (opB) size: 1"/>
</testcase>
<testcase name="compareQStringLists(short list first)" classname="tst_Cmptest" time="@TEST_DURATION@">
<failure type="fail" message="Compared lists have different sizes.
Actual (opA) size: 1
Expected (opB) size: 12"/>
</testcase>
<testcase name="compareQListInt(match)" classname="tst_Cmptest" time="@TEST_DURATION@"/>
<testcase name="compareQListInt(size mismatch)" classname="tst_Cmptest" time="@TEST_DURATION@">
<failure type="fail" message="Compared lists have different sizes.
Actual (actual) size: 2
Expected (expected) size: 3"/>
</testcase>
<testcase name="compareQListInt(value mismatch)" classname="tst_Cmptest" time="@TEST_DURATION@">
<failure type="fail" message="Compared lists differ at index 2.
Actual (actual): 4
Expected (expected): 3"/>
</testcase>
<testcase name="compareQListIntToArray(match)" classname="tst_Cmptest" time="@TEST_DURATION@"/>
<testcase name="compareQListIntToArray(size mismatch)" classname="tst_Cmptest" time="@TEST_DURATION@">
<failure type="fail" message="Compared lists have different sizes.
Actual (actual) size: 2
Expected (expected) size: 3"/>
</testcase>
<testcase name="compareQListIntToArray(value mismatch)" classname="tst_Cmptest" time="@TEST_DURATION@">
<failure type="fail" message="Compared lists differ at index 2.
Actual (actual): 4
Expected (expected): 3"/>
</testcase>
<testcase name="compareQListIntToInitializerList(match)" classname="tst_Cmptest" time="@TEST_DURATION@"/>
<testcase name="compareQListIntToInitializerList(size mismatch)" classname="tst_Cmptest" time="@TEST_DURATION@">
<failure type="fail" message="Compared lists have different sizes.
Actual (actual) size: 2
Expected (ARG({1, 2, 3})) size: 3"/>
</testcase>
<testcase name="compareQListIntToInitializerList(value mismatch)" classname="tst_Cmptest" time="@TEST_DURATION@">
<failure type="fail" message="Compared lists differ at index 2.
Actual (actual): 4
Expected (ARG({1, 2, 3})): 3"/>
</testcase>
<testcase name="compareQListDouble" classname="tst_Cmptest" time="@TEST_DURATION@">
<failure type="fail" message="Compared lists differ at index 0.
Actual (double1): 1.5
Expected (double2): 1"/>
</testcase>
<testcase name="compareQColor(Qt::yellow vs &quot;yellow&quot;)" classname="tst_Cmptest" time="@TEST_DURATION@"/>
<testcase name="compareQColor(Qt::yellow vs Qt::green)" classname="tst_Cmptest" time="@TEST_DURATION@">
<failure type="fail" message="Compared values are not the same
Actual (colorA): #ffffff00
Expected (colorB): #ff00ff00"/>
</testcase>
<testcase name="compareQColor(0x88ff0000 vs 0xffff0000)" classname="tst_Cmptest" time="@TEST_DURATION@">
<failure type="fail" message="Compared values are not the same
Actual (colorA): #88ff0000
Expected (colorB): #ffff0000"/>
</testcase>
<testcase name="compareQPixmaps(both null)" classname="tst_Cmptest" time="@TEST_DURATION@"/>
<testcase name="compareQPixmaps(one null)" classname="tst_Cmptest" time="@TEST_DURATION@">
<failure type="fail" message="Compared QPixmaps differ.
Actual (opA).isNull(): 1
Expected (opB).isNull(): 0"/>
</testcase>
<testcase name="compareQPixmaps(other null)" classname="tst_Cmptest" time="@TEST_DURATION@">
<failure type="fail" message="Compared QPixmaps differ.
Actual (opA).isNull(): 0
Expected (opB).isNull(): 1"/>
</testcase>
<testcase name="compareQPixmaps(equal)" classname="tst_Cmptest" time="@TEST_DURATION@"/>
<testcase name="compareQPixmaps(different size)" classname="tst_Cmptest" time="@TEST_DURATION@">
<failure type="fail" message="Compared QPixmaps differ in size.
Actual (opA): 11x20
Expected (opB): 20x20"/>
</testcase>
<testcase name="compareQPixmaps(different pixels)" classname="tst_Cmptest" time="@TEST_DURATION@">
<failure type="fail" message="Compared values are not the same"/>
</testcase>
<testcase name="compareQPixmaps(different dpr)" classname="tst_Cmptest" time="@TEST_DURATION@">
<failure type="fail" message="Compared QPixmaps differ in device pixel ratio.
Actual (opA): 1
Expected (opB): 2"/>
</testcase>
<testcase name="compareQImages(both null)" classname="tst_Cmptest" time="@TEST_DURATION@"/>
<testcase name="compareQImages(one null)" classname="tst_Cmptest" time="@TEST_DURATION@">
<failure type="fail" message="Compared QImages differ.
Actual (opA).isNull(): 1
Expected (opB).isNull(): 0"/>
</testcase>
<testcase name="compareQImages(other null)" classname="tst_Cmptest" time="@TEST_DURATION@">
<failure type="fail" message="Compared QImages differ.
Actual (opA).isNull(): 0
Expected (opB).isNull(): 1"/>
</testcase>
<testcase name="compareQImages(equal)" classname="tst_Cmptest" time="@TEST_DURATION@"/>
<testcase name="compareQImages(different size)" classname="tst_Cmptest" time="@TEST_DURATION@">
<failure type="fail" message="Compared QImages differ in size.
Actual (opA): 11x20
Expected (opB): 20x20"/>
</testcase>
<testcase name="compareQImages(different format)" classname="tst_Cmptest" time="@TEST_DURATION@">
<failure type="fail" message="Compared QImages differ in format.
Actual (opA): 6
Expected (opB): 3"/>
</testcase>
<testcase name="compareQImages(different pixels)" classname="tst_Cmptest" time="@TEST_DURATION@">
<failure type="fail" message="Compared values are not the same"/>
</testcase>
<testcase name="compareQImages(different dpr)" classname="tst_Cmptest" time="@TEST_DURATION@">
<failure type="fail" message="Compared QImages differ in device pixel ratio.
Actual (opA): 1
Expected (opB): 2"/>
</testcase>
<testcase name="compareQRegion(equal&#x002D;empty)" classname="tst_Cmptest" time="@TEST_DURATION@"/>
<testcase name="compareQRegion(1&#x002D;empty)" classname="tst_Cmptest" time="@TEST_DURATION@">
<failure type="fail" message="Compared values are not the same
Actual (rA): QRegion(200x50+10+10)
Expected (rB): QRegion(null)"/>
</testcase>
<testcase name="compareQRegion(equal)" classname="tst_Cmptest" time="@TEST_DURATION@"/>
<testcase name="compareQRegion(different lists)" classname="tst_Cmptest" time="@TEST_DURATION@">
<failure type="fail" message="Compared values are not the same
Actual (rA): QRegion(200x50+10+10)
Expected (rB): QRegion(2 rectangles, 50x200+100+200, 200x50+10+10)"/>
</testcase>
<testcase name="compareQVector2D" classname="tst_Cmptest" time="@TEST_DURATION@">
<failure type="fail" message="Compared values are not the same
Actual (v2a): QVector2D(1, 2)
Expected (v2b): QVector2D(1, 3)"/>
</testcase>
<testcase name="compareQVector3D" classname="tst_Cmptest" time="@TEST_DURATION@">
<failure type="fail" message="Compared values are not the same
Actual (v3a): QVector3D(1, 2, 3)
Expected (v3b): QVector3D(1, 3, 3)"/>
</testcase>
<testcase name="compareQVector4D" classname="tst_Cmptest" time="@TEST_DURATION@">
<failure type="fail" message="Compared values are not the same
Actual (v4a): QVector4D(1, 2, 3, 4)
Expected (v4b): QVector4D(1, 3, 3, 4)"/>
</testcase>
<testcase name="verify" classname="tst_Cmptest" time="@TEST_DURATION@">
<failure type="fail" message="&apos;opaqueFunc() &lt; 2&apos; returned FALSE. ()"/>
</testcase>
<testcase name="verify2" classname="tst_Cmptest" time="@TEST_DURATION@">
<failure type="fail" message="&apos;opaqueFunc() &lt; 2&apos; returned FALSE. (42)"/>
</testcase>
<testcase name="tryVerify" classname="tst_Cmptest" time="@TEST_DURATION@">
<failure type="fail" message="&apos;opaqueFunc() &lt; 2&apos; returned FALSE. ()"/>
</testcase>
<testcase name="tryVerify2" classname="tst_Cmptest" time="@TEST_DURATION@">
<failure type="fail" message="&apos;opaqueFunc() &lt; 2&apos; returned FALSE. (42)"/>
</testcase>
<testcase name="verifyExplicitOperatorBool" classname="tst_Cmptest" time="@TEST_DURATION@"/>
<testcase name="cleanupTestCase" classname="tst_Cmptest" time="@TEST_DURATION@"/>
<system-out/>
<system-err/>
</testsuite>
Reference in New Issue View Git Blame Copy Permalink