diff --git a/src/testlib/qtaptestlogger.cpp b/src/testlib/qtaptestlogger.cpp index e0c8feddaed..1b2a70a11d3 100644 --- a/src/testlib/qtaptestlogger.cpp +++ b/src/testlib/qtaptestlogger.cpp @@ -127,7 +127,6 @@ QT_BEGIN_NAMESPACE QTapTestLogger::QTapTestLogger(const char *filename) : QAbstractTestLogger(filename) - , m_wasExpectedFail(false) { } @@ -164,7 +163,7 @@ void QTapTestLogger::stopLogging() void QTapTestLogger::enterTestFunction(const char *function) { - m_wasExpectedFail = false; + m_firstExpectedFail.clear(); Q_ASSERT(!m_gatherMessages); Q_ASSERT(m_comments.isEmpty()); Q_ASSERT(m_messages.isEmpty()); @@ -173,7 +172,7 @@ void QTapTestLogger::enterTestFunction(const char *function) void QTapTestLogger::enterTestData(QTestData *data) { - m_wasExpectedFail = false; + m_firstExpectedFail.clear(); if (!m_messages.isEmpty() || !m_comments.isEmpty()) flushMessages(); m_gatherMessages = data != nullptr; @@ -250,22 +249,19 @@ void QTapTestLogger::flushMessages() void QTapTestLogger::addIncident(IncidentTypes type, const char *description, const char *file, int line) { - m_gatherMessages = false; - if (m_wasExpectedFail && (type == Pass || type == BlacklistedPass - || type == XFail || type == BlacklistedXFail)) { - // XFail comes with a corresponding Pass incident, but we only want - // to emit a single test point for it, so skip the this pass. - return; - } - m_wasExpectedFail = type == XFail || type == BlacklistedXFail; - const bool ok = type == Pass || type == BlacklistedPass || type == Skip - || type == XPass || type == BlacklistedXPass; + const bool isExpectedFail = type == XFail || type == BlacklistedXFail; + const bool ok = (m_firstExpectedFail.isEmpty() + && (type == Pass || type == BlacklistedPass || type == Skip + || type == XPass || type == BlacklistedXPass)); - const char *const incident = [type]() { + const char *const incident = [type](const char *priorXFail) { switch (type) { // We treat expected or blacklisted failures/passes as TODO-failures/passes, // which should be treated as soft issues by consumers. Not all do though :/ case BlacklistedPass: + if (priorXFail[0] != '\0') + return priorXFail; + Q_FALLTHROUGH(); case XFail: case BlacklistedXFail: case XPass: case BlacklistedXPass: case BlacklistedFail: @@ -273,116 +269,138 @@ void QTapTestLogger::addIncident(IncidentTypes type, const char *description, case Skip: return "SKIP"; case Pass: + if (priorXFail[0] != '\0') + return priorXFail; + Q_FALLTHROUGH(); case Fail: break; } return static_cast(nullptr); - }(); + }(m_firstExpectedFail.constData()); QTestCharBuffer directive; if (incident) { - QTest::qt_asprintf(&directive, " # %s%s%s", incident, + QTest::qt_asprintf(&directive, "%s%s%s%s", + isExpectedFail ? "" : " # ", incident, description && description[0] ? " " : "", description); } - int testNumber = QTestLog::totalCount(); - // That counts Pass, Fail, Skip and blacklisted; the XFail will eventually - // be added to it as a Pass, but that hasn't heppened yet, so count it now: - if (m_wasExpectedFail) - testNumber += 1; - - outputTestLine(ok, testNumber, directive); + if (!isExpectedFail) { + m_gatherMessages = false; + outputTestLine(ok, QTestLog::totalCount(), directive); + } else if (m_gatherMessages && m_firstExpectedFail.isEmpty()) { + QTestPrivate::appendCharBuffer(&m_firstExpectedFail, directive); + } flushComments(); if (!ok || !m_messages.isEmpty()) { // All failures need a diagnostics section to not confuse consumers. // We also need a diagnostics section when we have messages to report. - beginYamlish(); + if (isExpectedFail) { + QTestCharBuffer message; + if (m_gatherMessages) { + QTest::qt_asprintf(&message, YAML_INDENT YAML_INDENT "- severity: xfail\n" + YAML_INDENT YAML_INDENT YAML_INDENT "message:%s\n", + directive.constData() + 4); + } else { + QTest::qt_asprintf(&message, YAML_INDENT "# xfail:%s\n", directive.constData() + 4); + } + outputBuffer(message); + } else { + beginYamlish(); + } - if (!ok && type != XFail && type != BlacklistedXFail) { + if (!isExpectedFail || m_gatherMessages) { + const char *indent = isExpectedFail ? YAML_INDENT YAML_INDENT YAML_INDENT : YAML_INDENT; + if (!ok) { #if QT_CONFIG(regularexpression) - // This is fragile, but unfortunately testlib doesn't plumb - // the expected and actual values to the loggers (yet). - static QRegularExpression verifyRegex( - QLatin1String("^'(?.*)' returned (?\\w+).+\\((?.*)\\)$")); + // This is fragile, but unfortunately testlib doesn't plumb + // the expected and actual values to the loggers (yet). + static QRegularExpression verifyRegex( + QLatin1String("^'(?.*)' returned " + "(?\\w+).+\\((?.*)\\)$")); - static QRegularExpression comparRegex( - QLatin1String("^(?.*)\n" - "\\s*Actual\\s+\\((?.*)\\)\\s*: (?.*)\n" - "\\s*Expected\\s+\\((?.*)\\)\\s*: (?.*)$")); + static QRegularExpression compareRegex( + QLatin1String("^(?.*)\n" + "\\s*Actual\\s+\\((?.*)\\)\\s*: (?.*)\n" + "\\s*Expected\\s+\\((?.*)\\)\\s*: " + "(?.*)$")); - QString descriptionString = QString::fromUtf8(description); - QRegularExpressionMatch match = verifyRegex.match(descriptionString); - if (!match.hasMatch()) - match = comparRegex.match(descriptionString); + QString descriptionString = QString::fromUtf8(description); + QRegularExpressionMatch match = verifyRegex.match(descriptionString); + if (!match.hasMatch()) + match = compareRegex.match(descriptionString); - if (match.hasMatch()) { - bool isVerify = match.regularExpression() == verifyRegex; - QString message = match.captured(QLatin1String("message")); - QString expected; - QString actual; + if (match.hasMatch()) { + bool isVerify = match.regularExpression() == verifyRegex; + QString message = match.captured(QLatin1String("message")); + QString expected; + QString actual; - if (isVerify) { - QString expression = QLatin1String(" (") - % match.captured(QLatin1String("actualexpression")) % QLatin1Char(')') ; - actual = match.captured(QLatin1String("actual")).toLower() % expression; - expected = (actual.startsWith(QLatin1String("true")) ? QLatin1String("false") : QLatin1String("true")) % expression; - if (message.isEmpty()) - message = QLatin1String("Verification failed"); - } else { - expected = match.captured(QLatin1String("expected")) - % QLatin1String(" (") % match.captured(QLatin1String("expectedexpresssion")) % QLatin1Char(')'); - actual = match.captured(QLatin1String("actual")) - % QLatin1String(" (") % match.captured(QLatin1String("actualexpression")) % QLatin1Char(')'); - } + if (isVerify) { + QString expression = QLatin1String(" (") + % match.captured(QLatin1String("actualexpression")) % QLatin1Char(')') ; + actual = match.captured(QLatin1String("actual")).toLower() % expression; + expected = (actual.startsWith(QLatin1String("true")) + ? QLatin1String("false") + : QLatin1String("true")) % expression; + if (message.isEmpty()) + message = QLatin1String("Verification failed"); + } else { + expected = match.captured(QLatin1String("expected")) % QLatin1String(" (") + % match.captured(QLatin1String("expectedexpresssion")) + % QLatin1Char(')'); + actual = match.captured(QLatin1String("actual")) % QLatin1String(" (") + % match.captured(QLatin1String("actualexpression")) % QLatin1Char(')'); + } - QTestCharBuffer diagnosticsYamlish; - QTest::qt_asprintf(&diagnosticsYamlish, - YAML_INDENT "type: %s\n" - YAML_INDENT "message: %s\n" + QTestCharBuffer diagnosticsYamlish; + QTest::qt_asprintf(&diagnosticsYamlish, + "%stype: %s\n" + "%smessage: %s\n" + // Some consumers understand 'wanted/found', others need + // 'expected/actual', so be compatible with both. + "%swanted: %s\n" + "%sfound: %s\n" + "%sexpected: %s\n" + "%sactual: %s\n", + indent, isVerify ? "QVERIFY" : "QCOMPARE", + indent, qPrintable(message), + indent, qPrintable(expected), indent, qPrintable(actual), + indent, qPrintable(expected), indent, qPrintable(actual) + ); - // Some consumers understand 'wanted/found', while others need - // 'expected/actual', so we do both for maximum compatibility. - YAML_INDENT "wanted: %s\n" - YAML_INDENT "found: %s\n" - YAML_INDENT "expected: %s\n" - YAML_INDENT "actual: %s\n", - - isVerify ? "QVERIFY" : "QCOMPARE", - qPrintable(message), - qPrintable(expected), qPrintable(actual), - qPrintable(expected), qPrintable(actual) - ); - - outputBuffer(diagnosticsYamlish); - } else + outputBuffer(diagnosticsYamlish); + } else #endif - if (description && !incident) { - QTestCharBuffer unparsableDescription; - QTest::qt_asprintf(&unparsableDescription, YAML_INDENT "# %s\n", description); - outputBuffer(unparsableDescription); + if (description && !incident) { + QTestCharBuffer unparsableDescription; + QTest::qt_asprintf(&unparsableDescription, YAML_INDENT "# %s\n", description); + outputBuffer(unparsableDescription); + } + } + + if (file) { + QTestCharBuffer location; + QTest::qt_asprintf(&location, + // The generic 'at' key is understood by most consumers. + "%sat: %s::%s() (%s:%d)\n" + + // The file and line keys are for consumers that are able + // to read more granular location info. + "%sfile: %s\n" + "%sline: %d\n", + + indent, QTestResult::currentTestObjectName(), + QTestResult::currentTestFunction(), + file, line, indent, file, indent, line + ); + outputBuffer(location); } } - if (file) { - QTestCharBuffer location; - QTest::qt_asprintf(&location, - // The generic 'at' key is understood by most consumers. - YAML_INDENT "at: %s::%s() (%s:%d)\n" - - // The file and line keys are for consumers that are able - // to read more granular location info. - YAML_INDENT "file: %s\n" - YAML_INDENT "line: %d\n", - - QTestResult::currentTestObjectName(), - QTestResult::currentTestFunction(), - file, line, file, line - ); - outputBuffer(location); - } - - endYamlish(); + if (!isExpectedFail) + endYamlish(); } } diff --git a/src/testlib/qtaptestlogger_p.h b/src/testlib/qtaptestlogger_p.h index 2650e958959..a9fafeb6b96 100644 --- a/src/testlib/qtaptestlogger_p.h +++ b/src/testlib/qtaptestlogger_p.h @@ -82,7 +82,7 @@ private: void flushMessages(); void beginYamlish(); void endYamlish(); - bool m_wasExpectedFail; + QTestCharBuffer m_firstExpectedFail; QTestCharBuffer m_comments; QTestCharBuffer m_messages; bool m_gatherMessages = false; diff --git a/tests/auto/testlib/selftests/expected_blacklisted.tap b/tests/auto/testlib/selftests/expected_blacklisted.tap index 3217b42a4f6..83def09e348 100644 --- a/tests/auto/testlib/selftests/expected_blacklisted.tap +++ b/tests/auto/testlib/selftests/expected_blacklisted.tap @@ -23,9 +23,13 @@ not ok 4 - fail() # TODO 'false' returned FALSE. (This test should BFAIL) ... not ok 5 - xfail() # TODO This test should BXFAIL then BPASS --- - at: tst_Blacklisted::xfail() (qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp:0) - file: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp - line: 0 + extensions: + messages: + - severity: xfail + message: This test should BXFAIL then BPASS + at: tst_Blacklisted::xfail() (qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp:0) + file: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp + line: 0 ... ok 6 - multiSkip() # SKIP This skip should be repeated ten times ok 6 - multiSkip() # SKIP This skip should be repeated ten times @@ -104,24 +108,31 @@ not ok 7 - multiFail() # TODO But this test should only contribute one to the bl file: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp line: 0 ... -not ok 8 - xfailContinueSkip() # TODO This test should BXFAIL then SKIP +not ok 8 - xfailContinueSkip() # SKIP This skip should be seen and counted --- at: tst_Blacklisted::xfailContinueSkip() (qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp:0) file: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp line: 0 - ... -ok 8 - xfailContinueSkip() # SKIP This skip should be seen and counted -not ok 9 - xfailContinueFail() # TODO This test should BXFAIL then BFAIL - --- - at: tst_Blacklisted::xfailContinueFail() (qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp:0) - file: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp - line: 0 + extensions: + messages: + - severity: xfail + message: This test should BXFAIL then SKIP + at: tst_Blacklisted::xfailContinueSkip() (qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp:0) + file: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp + line: 0 ... not ok 9 - xfailContinueFail() # TODO This fail should be seen and counted as blacklisted --- at: tst_Blacklisted::xfailContinueFail() (qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp:0) file: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp line: 0 + extensions: + messages: + - severity: xfail + message: This test should BXFAIL then BFAIL + at: tst_Blacklisted::xfailContinueFail() (qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp:0) + file: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp + line: 0 ... ok 10 - xpass() # TODO 'true' returned TRUE unexpectedly. (This test should BXPASS) ok 11 - xpassContinueSkip() # TODO 'true' returned TRUE unexpectedly. (This test should BXPASS then SKIP) diff --git a/tests/auto/testlib/selftests/expected_expectfail.tap b/tests/auto/testlib/selftests/expected_expectfail.tap index c61f47fd480..8ad34c5f0ad 100644 --- a/tests/auto/testlib/selftests/expected_expectfail.tap +++ b/tests/auto/testlib/selftests/expected_expectfail.tap @@ -3,37 +3,52 @@ TAP version 13 ok 1 - initTestCase() not ok 2 - xfailAndContinue() # TODO This should xfail --- - at: tst_ExpectFail::xfailAndContinue() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0) - file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp - line: 0 extensions: messages: - severity: debug message: begin + - severity: xfail + message: This should xfail + at: tst_ExpectFail::xfailAndContinue() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0) + file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp + line: 0 + - severity: debug + message: after ... -# debug: after not ok 3 - xfailAndAbort() # TODO This should xfail --- - at: tst_ExpectFail::xfailAndAbort() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0) - file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp - line: 0 extensions: messages: - severity: debug message: begin + - severity: xfail + message: This should xfail + at: tst_ExpectFail::xfailAndAbort() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0) + file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp + line: 0 ... -not ok 4 - xfailContinueSkip() # TODO This should xfail then skip +not ok 4 - xfailContinueSkip() # SKIP This skip should be reported and counted --- at: tst_ExpectFail::xfailContinueSkip() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0) file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp line: 0 + extensions: + messages: + - severity: xfail + message: This should xfail then skip + at: tst_ExpectFail::xfailContinueSkip() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0) + file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp + line: 0 ... -ok 4 - xfailContinueSkip() # SKIP This skip should be reported and counted not ok 5 - xfailAbortSkip() # TODO This should xfail --- - at: tst_ExpectFail::xfailAbortSkip() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0) - file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp - line: 0 + extensions: + messages: + - severity: xfail + message: This should xfail + at: tst_ExpectFail::xfailAbortSkip() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0) + file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp + line: 0 ... not ok 6 - xfailTwice() --- @@ -72,52 +87,92 @@ not ok 10 - xfailDataDrivenTwice(Fail Continue) ... not ok 11 - xfailWithQString() # TODO A string --- - at: tst_ExpectFail::xfailWithQString() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0) - file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp - line: 0 + extensions: + messages: + - severity: xfail + message: A string + at: tst_ExpectFail::xfailWithQString() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0) + file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp + line: 0 + - severity: xfail + message: Bug 5 (The message) + at: tst_ExpectFail::xfailWithQString() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0) + file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp + line: 0 ... ok 12 - xfailDataDrivenWithQString(Pass Abort) # SKIP Each Continue or Pass reports this and increments skip-count ok 13 - xfailDataDrivenWithQString(Pass Continue) # SKIP Each Continue or Pass reports this and increments skip-count not ok 14 - xfailDataDrivenWithQString(Fail Abort) # TODO A string --- - at: tst_ExpectFail::xfailDataDrivenWithQString() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0) - file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp - line: 0 + extensions: + messages: + - severity: xfail + message: A string + at: tst_ExpectFail::xfailDataDrivenWithQString() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0) + file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp + line: 0 ... -not ok 15 - xfailDataDrivenWithQString(Fail Continue) # TODO A string +not ok 15 - xfailDataDrivenWithQString(Fail Continue) # SKIP Each Continue or Pass reports this and increments skip-count --- at: tst_ExpectFail::xfailDataDrivenWithQString() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0) file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp line: 0 + extensions: + messages: + - severity: xfail + message: A string + at: tst_ExpectFail::xfailDataDrivenWithQString() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0) + file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp + line: 0 + - severity: xfail + message: Bug 5 (The message) + at: tst_ExpectFail::xfailDataDrivenWithQString() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0) + file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp + line: 0 ... -ok 15 - xfailDataDrivenWithQString(Fail Continue) # SKIP Each Continue or Pass reports this and increments skip-count ok 16 - xfailDataDrivenWithQVerify(Pass Abort) ok 17 - xfailDataDrivenWithQVerify(Pass Continue) not ok 18 - xfailDataDrivenWithQVerify(Fail Abort) # TODO This test should xfail --- - at: tst_ExpectFail::xfailDataDrivenWithQVerify() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0) - file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp - line: 0 + extensions: + messages: + - severity: xfail + message: This test should xfail + at: tst_ExpectFail::xfailDataDrivenWithQVerify() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0) + file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp + line: 0 ... not ok 19 - xfailDataDrivenWithQVerify(Fail Continue) # TODO This test should xfail --- - at: tst_ExpectFail::xfailDataDrivenWithQVerify() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0) - file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp - line: 0 + extensions: + messages: + - severity: xfail + message: This test should xfail + at: tst_ExpectFail::xfailDataDrivenWithQVerify() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0) + file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp + line: 0 ... ok 20 - xfailDataDrivenWithQCompare(Pass Abort) ok 21 - xfailDataDrivenWithQCompare(Pass Continue) not ok 22 - xfailDataDrivenWithQCompare(Fail Abort) # TODO This test should xfail --- - at: tst_ExpectFail::xfailDataDrivenWithQCompare() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0) - file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp - line: 0 + extensions: + messages: + - severity: xfail + message: This test should xfail + at: tst_ExpectFail::xfailDataDrivenWithQCompare() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0) + file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp + line: 0 ... not ok 23 - xfailDataDrivenWithQCompare(Fail Continue) # TODO This test should xfail --- - at: tst_ExpectFail::xfailDataDrivenWithQCompare() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0) - file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp - line: 0 + extensions: + messages: + - severity: xfail + message: This test should xfail + at: tst_ExpectFail::xfailDataDrivenWithQCompare() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0) + file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp + line: 0 ... ok 24 - xfailOnWrongRow(Fail Abort) --- @@ -135,15 +190,23 @@ ok 25 - xfailOnWrongRow(Fail Continue) ... not ok 26 - xfailOnAnyRow(Fail Abort) # TODO This test should xfail --- - at: tst_ExpectFail::xfailOnAnyRow() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0) - file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp - line: 0 + extensions: + messages: + - severity: xfail + message: This test should xfail + at: tst_ExpectFail::xfailOnAnyRow() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0) + file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp + line: 0 ... not ok 27 - xfailOnAnyRow(Fail Continue) # TODO This test should xfail --- - at: tst_ExpectFail::xfailOnAnyRow() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0) - file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp - line: 0 + extensions: + messages: + - severity: xfail + message: This test should xfail + at: tst_ExpectFail::xfailOnAnyRow() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0) + file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp + line: 0 ... not ok 28 - xfailWithoutCheck(Fail Abort) --- @@ -169,12 +232,7 @@ ok 33 - xpassContinue() # TODO 'true' returned TRUE unexpectedly. () ok 34 - xpassContinueSkip() # TODO 'true' returned TRUE unexpectedly. () ok 34 - xpassContinueSkip() # SKIP This should be reached but not increment skip-count ok 35 - xpassContinueXfailAbort() # TODO 'true' returned TRUE unexpectedly. () -not ok 36 - xpassContinueXfailAbort() # TODO This test should xfail but not add to totals - --- - at: tst_ExpectFail::xpassContinueXfailAbort() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0) - file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp - line: 0 - ... + # xfail: This test should xfail but not add to totals ok 36 - xpassAbortDataDrivenWithQVerify(XPass) # TODO 'true' returned TRUE unexpectedly. () ok 37 - xpassAbortDataDrivenWithQVerify(Pass) ok 38 - xpassContinueDataDrivenWithQVerify(XPass) # TODO 'true' returned TRUE unexpectedly. () diff --git a/tests/auto/testlib/selftests/expected_strcmp.tap b/tests/auto/testlib/selftests/expected_strcmp.tap index 0ff00d9b671..8f58ec8d244 100644 --- a/tests/auto/testlib/selftests/expected_strcmp.tap +++ b/tests/auto/testlib/selftests/expected_strcmp.tap @@ -2,12 +2,6 @@ TAP version 13 # tst_StrCmp ok 1 - initTestCase() ok 2 - compareCharStars() -not ok 3 - compareByteArray() # TODO Next test should fail - --- - at: tst_StrCmp::compareByteArray() (qtbase/tests/auto/testlib/selftests/strcmp/tst_strcmp.cpp:0) - file: qtbase/tests/auto/testlib/selftests/strcmp/tst_strcmp.cpp - line: 0 - ... not ok 3 - compareByteArray() --- type: QCOMPARE @@ -19,6 +13,23 @@ not ok 3 - compareByteArray() at: tst_StrCmp::compareByteArray() (qtbase/tests/auto/testlib/selftests/strcmp/tst_strcmp.cpp:0) file: qtbase/tests/auto/testlib/selftests/strcmp/tst_strcmp.cpp line: 0 + extensions: + messages: + - severity: xfail + message: Next test should fail + at: tst_StrCmp::compareByteArray() (qtbase/tests/auto/testlib/selftests/strcmp/tst_strcmp.cpp:0) + file: qtbase/tests/auto/testlib/selftests/strcmp/tst_strcmp.cpp + line: 0 + - severity: xfail + message: Next test should fail + at: tst_StrCmp::compareByteArray() (qtbase/tests/auto/testlib/selftests/strcmp/tst_strcmp.cpp:0) + file: qtbase/tests/auto/testlib/selftests/strcmp/tst_strcmp.cpp + line: 0 + - severity: xfail + message: Next test should fail + at: tst_StrCmp::compareByteArray() (qtbase/tests/auto/testlib/selftests/strcmp/tst_strcmp.cpp:0) + file: qtbase/tests/auto/testlib/selftests/strcmp/tst_strcmp.cpp + line: 0 ... not ok 4 - failByteArray() ---