qiurui
/
qtbase
mirror of https://github.com/qt/qtbase.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

TAP test logger: report B?XFAIL (mostly) as a message 

Previously, only the first B?XFAIL would be reported, all others would
be discarded. Furthermore, if a B?XFAIL had happened, B?PASS was also
not reported, since the B?XFAIL served as test line. However, if the
B?XFAIL was followed by a SKIP, B?XPASS or B?FAIL, these were reported
as normal, producing exactly the kind of duplicated test line that the
skipping of B?PASS was meant to supply.

So change B?XFAIL to be reported among the messages, but retain the
TODO annotation of the first on the test line of a subsequent B?PASS,
if nothing more drastic happens in the mean time. So now more than one
B?XFAIL can be reported, the test is still marked as a TODO and we
don't get duplicate test lines for a subsequent non-passing result.

This replaces the bool m_wasExpectedFail member with a QTestCharBuffer
m_firstExpectedFail that records the first XFAIL's TODO line (so its
isEmpty() fully replaces m_wasExpectedFail).

Previously, the at/file/line information for a B?XFail would be
supplied as top-level keys in the YAML block for a "Pass" reported as
not ok due to the XFail, as this location information is now part of
the B?XFail's message in the extensions/messages block. Duplicating
the first B?XFail's location at top level would add complexity and is
arguably misleading, as the test result is really a pass (after
ignoring known issues), and the location of the pass is indeterminate
(nominally the end of the test function, but actually also after the
cleanup() call for this test, when relevant), which is why a Pass has
no location information.

Task-number: QTBUG-96844
Change-Id: Ib3f24f56266ff49bf3bc9759ac8263ac69a62130
Reviewed-by: Tor Arne Vestbø <tor.arne.vestbo@qt.io>
This commit is contained in:
Edward Welbourne 2022-03-14 19:22:18 +01:00
parent ac38401d31
commit 784f5d76b9
5 changed files with 257 additions and 159 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

212
src/testlib/qtaptestlogger.cpp
View File

@ -127,7 +127,6 @@ QT_BEGIN_NAMESPACE
QTapTestLogger::QTapTestLogger(const char *filename) QTapTestLogger::QTapTestLogger(const char *filename)
: QAbstractTestLogger(filename) : QAbstractTestLogger(filename)
, m_wasExpectedFail(false)
{ {
} }
@ -164,7 +163,7 @@ void QTapTestLogger::stopLogging()
void QTapTestLogger::enterTestFunction(const char *function) void QTapTestLogger::enterTestFunction(const char *function)
{ {
m_wasExpectedFail = false; m_firstExpectedFail.clear();
Q_ASSERT(!m_gatherMessages); Q_ASSERT(!m_gatherMessages);
Q_ASSERT(m_comments.isEmpty()); Q_ASSERT(m_comments.isEmpty());
Q_ASSERT(m_messages.isEmpty()); Q_ASSERT(m_messages.isEmpty());
@ -173,7 +172,7 @@ void QTapTestLogger::enterTestFunction(const char *function)
void QTapTestLogger::enterTestData(QTestData *data) void QTapTestLogger::enterTestData(QTestData *data)
{ {
m_wasExpectedFail = false; m_firstExpectedFail.clear();
if (!m_messages.isEmpty() || !m_comments.isEmpty()) if (!m_messages.isEmpty() || !m_comments.isEmpty())
flushMessages(); flushMessages();
m_gatherMessages = data != nullptr; m_gatherMessages = data != nullptr;
@ -250,22 +249,19 @@ void QTapTestLogger::flushMessages()
void QTapTestLogger::addIncident(IncidentTypes type, const char *description, void QTapTestLogger::addIncident(IncidentTypes type, const char *description,
const char *file, int line) const char *file, int line)
{ {
m_gatherMessages = false; const bool isExpectedFail = type == XFail || type == BlacklistedXFail;
if (m_wasExpectedFail && (type == Pass || type == BlacklistedPass const bool ok = (m_firstExpectedFail.isEmpty()
|| type == XFail || type == BlacklistedXFail)) { && (type == Pass || type == BlacklistedPass || type == Skip
// XFail comes with a corresponding Pass incident, but we only want || type == XPass || type == BlacklistedXPass));
// to emit a single test point for it, so skip the this pass.
return;
}
m_wasExpectedFail = type == XFail || type == BlacklistedXFail;
const bool ok = type == Pass || type == BlacklistedPass || type == Skip
|| type == XPass || type == BlacklistedXPass;
const char *const incident = [type]() { const char *const incident = [type](const char *priorXFail) {
switch (type) { switch (type) {
// We treat expected or blacklisted failures/passes as TODO-failures/passes, // We treat expected or blacklisted failures/passes as TODO-failures/passes,
// which should be treated as soft issues by consumers. Not all do though :/ // which should be treated as soft issues by consumers. Not all do though :/
case BlacklistedPass: case BlacklistedPass:
if (priorXFail[0] != '\0')
return priorXFail;
Q_FALLTHROUGH();
case XFail: case BlacklistedXFail: case XFail: case BlacklistedXFail:
case XPass: case BlacklistedXPass: case XPass: case BlacklistedXPass:
case BlacklistedFail: case BlacklistedFail:
@ -273,116 +269,138 @@ void QTapTestLogger::addIncident(IncidentTypes type, const char *description,
case Skip: case Skip:
return "SKIP"; return "SKIP";
case Pass: case Pass:
if (priorXFail[0] != '\0')
return priorXFail;
Q_FALLTHROUGH();
case Fail: case Fail:
break; break;
} }
return static_cast<const char *>(nullptr); return static_cast<const char *>(nullptr);
}(); }(m_firstExpectedFail.constData());
QTestCharBuffer directive; QTestCharBuffer directive;
if (incident) { if (incident) {
QTest::qt_asprintf(&directive, " # %s%s%s", incident, QTest::qt_asprintf(&directive, "%s%s%s%s",
isExpectedFail ? "" : " # ", incident,
description && description[0] ? " " : "", description); description && description[0] ? " " : "", description);
} }
int testNumber = QTestLog::totalCount(); if (!isExpectedFail) {
// That counts Pass, Fail, Skip and blacklisted; the XFail will eventually m_gatherMessages = false;
// be added to it as a Pass, but that hasn't heppened yet, so count it now: outputTestLine(ok, QTestLog::totalCount(), directive);
if (m_wasExpectedFail) } else if (m_gatherMessages && m_firstExpectedFail.isEmpty()) {
testNumber += 1; QTestPrivate::appendCharBuffer(&m_firstExpectedFail, directive);
}
outputTestLine(ok, testNumber, directive);
flushComments(); flushComments();
if (!ok || !m_messages.isEmpty()) { if (!ok || !m_messages.isEmpty()) {
// All failures need a diagnostics section to not confuse consumers. // All failures need a diagnostics section to not confuse consumers.
// We also need a diagnostics section when we have messages to report. // We also need a diagnostics section when we have messages to report.
beginYamlish(); if (isExpectedFail) {
QTestCharBuffer message;
if (m_gatherMessages) {
QTest::qt_asprintf(&message, YAML_INDENT YAML_INDENT "- severity: xfail\n"
YAML_INDENT YAML_INDENT YAML_INDENT "message:%s\n",
directive.constData() + 4);
} else {
QTest::qt_asprintf(&message, YAML_INDENT "# xfail:%s\n", directive.constData() + 4);
}
outputBuffer(message);
} else {
beginYamlish();
}
if (!ok && type != XFail && type != BlacklistedXFail) { if (!isExpectedFail || m_gatherMessages) {
const char *indent = isExpectedFail ? YAML_INDENT YAML_INDENT YAML_INDENT : YAML_INDENT;
if (!ok) {
#if QT_CONFIG(regularexpression) #if QT_CONFIG(regularexpression)
// This is fragile, but unfortunately testlib doesn't plumb // This is fragile, but unfortunately testlib doesn't plumb
// the expected and actual values to the loggers (yet). // the expected and actual values to the loggers (yet).
static QRegularExpression verifyRegex( static QRegularExpression verifyRegex(
QLatin1String("^'(?<actualexpression>.*)' returned (?<actual>\\w+).+\\((?<message>.*)\\)$")); QLatin1String("^'(?<actualexpression>.*)' returned "
"(?<actual>\\w+).+\\((?<message>.*)\\)$"));
static QRegularExpression comparRegex( static QRegularExpression compareRegex(
QLatin1String("^(?<message>.*)\n" QLatin1String("^(?<message>.*)\n"
"\\s*Actual\\s+\\((?<actualexpression>.*)\\)\\s*: (?<actual>.*)\n" "\\s*Actual\\s+\\((?<actualexpression>.*)\\)\\s*: (?<actual>.*)\n"
"\\s*Expected\\s+\\((?<expectedexpresssion>.*)\\)\\s*: (?<expected>.*)$")); "\\s*Expected\\s+\\((?<expectedexpresssion>.*)\\)\\s*: "
"(?<expected>.*)$"));
QString descriptionString = QString::fromUtf8(description); QString descriptionString = QString::fromUtf8(description);
QRegularExpressionMatch match = verifyRegex.match(descriptionString); QRegularExpressionMatch match = verifyRegex.match(descriptionString);
if (!match.hasMatch()) if (!match.hasMatch())
match = comparRegex.match(descriptionString); match = compareRegex.match(descriptionString);
if (match.hasMatch()) { if (match.hasMatch()) {
bool isVerify = match.regularExpression() == verifyRegex; bool isVerify = match.regularExpression() == verifyRegex;
QString message = match.captured(QLatin1String("message")); QString message = match.captured(QLatin1String("message"));
QString expected; QString expected;
QString actual; QString actual;
if (isVerify) { if (isVerify) {
QString expression = QLatin1String(" (") QString expression = QLatin1String(" (")
% match.captured(QLatin1String("actualexpression")) % QLatin1Char(')') ; % match.captured(QLatin1String("actualexpression")) % QLatin1Char(')') ;
actual = match.captured(QLatin1String("actual")).toLower() % expression; actual = match.captured(QLatin1String("actual")).toLower() % expression;
expected = (actual.startsWith(QLatin1String("true")) ? QLatin1String("false") : QLatin1String("true")) % expression; expected = (actual.startsWith(QLatin1String("true"))
if (message.isEmpty()) ? QLatin1String("false")
message = QLatin1String("Verification failed"); : QLatin1String("true")) % expression;
} else { if (message.isEmpty())
expected = match.captured(QLatin1String("expected")) message = QLatin1String("Verification failed");
% QLatin1String(" (") % match.captured(QLatin1String("expectedexpresssion")) % QLatin1Char(')'); } else {
actual = match.captured(QLatin1String("actual")) expected = match.captured(QLatin1String("expected")) % QLatin1String(" (")
% QLatin1String(" (") % match.captured(QLatin1String("actualexpression")) % QLatin1Char(')'); % match.captured(QLatin1String("expectedexpresssion"))
} % QLatin1Char(')');
actual = match.captured(QLatin1String("actual")) % QLatin1String(" (")
% match.captured(QLatin1String("actualexpression")) % QLatin1Char(')');
}
QTestCharBuffer diagnosticsYamlish; QTestCharBuffer diagnosticsYamlish;
QTest::qt_asprintf(&diagnosticsYamlish, QTest::qt_asprintf(&diagnosticsYamlish,
YAML_INDENT "type: %s\n" "%stype: %s\n"
YAML_INDENT "message: %s\n" "%smessage: %s\n"
// Some consumers understand 'wanted/found', others need
// 'expected/actual', so be compatible with both.
"%swanted: %s\n"
"%sfound: %s\n"
"%sexpected: %s\n"
"%sactual: %s\n",
indent, isVerify ? "QVERIFY" : "QCOMPARE",
indent, qPrintable(message),
indent, qPrintable(expected), indent, qPrintable(actual),
indent, qPrintable(expected), indent, qPrintable(actual)
);
// Some consumers understand 'wanted/found', while others need outputBuffer(diagnosticsYamlish);
// 'expected/actual', so we do both for maximum compatibility. } else
YAML_INDENT "wanted: %s\n"
YAML_INDENT "found: %s\n"
YAML_INDENT "expected: %s\n"
YAML_INDENT "actual: %s\n",
isVerify ? "QVERIFY" : "QCOMPARE",
qPrintable(message),
qPrintable(expected), qPrintable(actual),
qPrintable(expected), qPrintable(actual)
);
outputBuffer(diagnosticsYamlish);
} else
#endif #endif
if (description && !incident) { if (description && !incident) {
QTestCharBuffer unparsableDescription; QTestCharBuffer unparsableDescription;
QTest::qt_asprintf(&unparsableDescription, YAML_INDENT "# %s\n", description); QTest::qt_asprintf(&unparsableDescription, YAML_INDENT "# %s\n", description);
outputBuffer(unparsableDescription); outputBuffer(unparsableDescription);
}
}
if (file) {
QTestCharBuffer location;
QTest::qt_asprintf(&location,
// The generic 'at' key is understood by most consumers.
"%sat: %s::%s() (%s:%d)\n"
// The file and line keys are for consumers that are able
// to read more granular location info.
"%sfile: %s\n"
"%sline: %d\n",
indent, QTestResult::currentTestObjectName(),
QTestResult::currentTestFunction(),
file, line, indent, file, indent, line
);
outputBuffer(location);
} }
} }
if (file) { if (!isExpectedFail)
QTestCharBuffer location; endYamlish();
QTest::qt_asprintf(&location,
// The generic 'at' key is understood by most consumers.
YAML_INDENT "at: %s::%s() (%s:%d)\n"
// The file and line keys are for consumers that are able
// to read more granular location info.
YAML_INDENT "file: %s\n"
YAML_INDENT "line: %d\n",
QTestResult::currentTestObjectName(),
QTestResult::currentTestFunction(),
file, line, file, line
);
outputBuffer(location);
}
endYamlish();
} }
} }

2
src/testlib/qtaptestlogger_p.h
View File

@ -82,7 +82,7 @@ private:
void flushMessages(); void flushMessages();
void beginYamlish(); void beginYamlish();
void endYamlish(); void endYamlish();
bool m_wasExpectedFail; QTestCharBuffer m_firstExpectedFail;
QTestCharBuffer m_comments; QTestCharBuffer m_comments;
QTestCharBuffer m_messages; QTestCharBuffer m_messages;
bool m_gatherMessages = false; bool m_gatherMessages = false;

33
tests/auto/testlib/selftests/expected_blacklisted.tap
View File

@ -23,9 +23,13 @@ not ok 4 - fail() # TODO 'false' returned FALSE. (This test should BFAIL)
... ...
not ok 5 - xfail() # TODO This test should BXFAIL then BPASS not ok 5 - xfail() # TODO This test should BXFAIL then BPASS
--- ---
at: tst_Blacklisted::xfail() (qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp:0) extensions:
file: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp messages:
line: 0 - severity: xfail
message: This test should BXFAIL then BPASS
at: tst_Blacklisted::xfail() (qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp:0)
file: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp
line: 0
... ...
ok 6 - multiSkip() # SKIP This skip should be repeated ten times ok 6 - multiSkip() # SKIP This skip should be repeated ten times
ok 6 - multiSkip() # SKIP This skip should be repeated ten times ok 6 - multiSkip() # SKIP This skip should be repeated ten times
@ -104,24 +108,31 @@ not ok 7 - multiFail() # TODO But this test should only contribute one to the bl
file: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp file: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp
line: 0 line: 0
... ...
not ok 8 - xfailContinueSkip() # TODO This test should BXFAIL then SKIP not ok 8 - xfailContinueSkip() # SKIP This skip should be seen and counted
--- ---
at: tst_Blacklisted::xfailContinueSkip() (qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp:0) at: tst_Blacklisted::xfailContinueSkip() (qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp:0)
file: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp file: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp
line: 0 line: 0
... extensions:
ok 8 - xfailContinueSkip() # SKIP This skip should be seen and counted messages:
not ok 9 - xfailContinueFail() # TODO This test should BXFAIL then BFAIL - severity: xfail
--- message: This test should BXFAIL then SKIP
at: tst_Blacklisted::xfailContinueFail() (qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp:0) at: tst_Blacklisted::xfailContinueSkip() (qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp:0)
file: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp file: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp
line: 0 line: 0
... ...
not ok 9 - xfailContinueFail() # TODO This fail should be seen and counted as blacklisted not ok 9 - xfailContinueFail() # TODO This fail should be seen and counted as blacklisted
--- ---
at: tst_Blacklisted::xfailContinueFail() (qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp:0) at: tst_Blacklisted::xfailContinueFail() (qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp:0)
file: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp file: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp
line: 0 line: 0
extensions:
messages:
- severity: xfail
message: This test should BXFAIL then BFAIL
at: tst_Blacklisted::xfailContinueFail() (qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp:0)
file: qtbase/tests/auto/testlib/selftests/blacklisted/tst_blacklisted.cpp
line: 0
... ...
ok 10 - xpass() # TODO 'true' returned TRUE unexpectedly. (This test should BXPASS) ok 10 - xpass() # TODO 'true' returned TRUE unexpectedly. (This test should BXPASS)
ok 11 - xpassContinueSkip() # TODO 'true' returned TRUE unexpectedly. (This test should BXPASS then SKIP) ok 11 - xpassContinueSkip() # TODO 'true' returned TRUE unexpectedly. (This test should BXPASS then SKIP)

146
tests/auto/testlib/selftests/expected_expectfail.tap
View File

@ -3,37 +3,52 @@ TAP version 13
ok 1 - initTestCase() ok 1 - initTestCase()
not ok 2 - xfailAndContinue() # TODO This should xfail not ok 2 - xfailAndContinue() # TODO This should xfail
--- ---
at: tst_ExpectFail::xfailAndContinue() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0)
file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
line: 0
extensions: extensions:
messages: messages:
- severity: debug - severity: debug
message: begin message: begin
- severity: xfail
message: This should xfail
at: tst_ExpectFail::xfailAndContinue() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0)
file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
line: 0
- severity: debug
message: after
... ...
# debug: after
not ok 3 - xfailAndAbort() # TODO This should xfail not ok 3 - xfailAndAbort() # TODO This should xfail
--- ---
at: tst_ExpectFail::xfailAndAbort() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0)
file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
line: 0
extensions: extensions:
messages: messages:
- severity: debug - severity: debug
message: begin message: begin
- severity: xfail
message: This should xfail
at: tst_ExpectFail::xfailAndAbort() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0)
file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
line: 0
... ...
not ok 4 - xfailContinueSkip() # TODO This should xfail then skip not ok 4 - xfailContinueSkip() # SKIP This skip should be reported and counted
--- ---
at: tst_ExpectFail::xfailContinueSkip() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0) at: tst_ExpectFail::xfailContinueSkip() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0)
file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
line: 0 line: 0
extensions:
messages:
- severity: xfail
message: This should xfail then skip
at: tst_ExpectFail::xfailContinueSkip() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0)
file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
line: 0
... ...
ok 4 - xfailContinueSkip() # SKIP This skip should be reported and counted
not ok 5 - xfailAbortSkip() # TODO This should xfail not ok 5 - xfailAbortSkip() # TODO This should xfail
--- ---
at: tst_ExpectFail::xfailAbortSkip() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0) extensions:
file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp messages:
line: 0 - severity: xfail
message: This should xfail
at: tst_ExpectFail::xfailAbortSkip() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0)
file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
line: 0
... ...
not ok 6 - xfailTwice() not ok 6 - xfailTwice()
--- ---
@ -72,52 +87,92 @@ not ok 10 - xfailDataDrivenTwice(Fail Continue)
... ...
not ok 11 - xfailWithQString() # TODO A string not ok 11 - xfailWithQString() # TODO A string
--- ---
at: tst_ExpectFail::xfailWithQString() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0) extensions:
file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp messages:
line: 0 - severity: xfail
message: A string
at: tst_ExpectFail::xfailWithQString() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0)
file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
line: 0
- severity: xfail
message: Bug 5 (The message)
at: tst_ExpectFail::xfailWithQString() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0)
file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
line: 0
... ...
ok 12 - xfailDataDrivenWithQString(Pass Abort) # SKIP Each Continue or Pass reports this and increments skip-count ok 12 - xfailDataDrivenWithQString(Pass Abort) # SKIP Each Continue or Pass reports this and increments skip-count
ok 13 - xfailDataDrivenWithQString(Pass Continue) # SKIP Each Continue or Pass reports this and increments skip-count ok 13 - xfailDataDrivenWithQString(Pass Continue) # SKIP Each Continue or Pass reports this and increments skip-count
not ok 14 - xfailDataDrivenWithQString(Fail Abort) # TODO A string not ok 14 - xfailDataDrivenWithQString(Fail Abort) # TODO A string
--- ---
at: tst_ExpectFail::xfailDataDrivenWithQString() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0) extensions:
file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp messages:
line: 0 - severity: xfail
message: A string
at: tst_ExpectFail::xfailDataDrivenWithQString() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0)
file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
line: 0
... ...
not ok 15 - xfailDataDrivenWithQString(Fail Continue) # TODO A string not ok 15 - xfailDataDrivenWithQString(Fail Continue) # SKIP Each Continue or Pass reports this and increments skip-count
--- ---
at: tst_ExpectFail::xfailDataDrivenWithQString() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0) at: tst_ExpectFail::xfailDataDrivenWithQString() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0)
file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
line: 0 line: 0
extensions:
messages:
- severity: xfail
message: A string
at: tst_ExpectFail::xfailDataDrivenWithQString() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0)
file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
line: 0
- severity: xfail
message: Bug 5 (The message)
at: tst_ExpectFail::xfailDataDrivenWithQString() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0)
file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
line: 0
... ...
ok 15 - xfailDataDrivenWithQString(Fail Continue) # SKIP Each Continue or Pass reports this and increments skip-count
ok 16 - xfailDataDrivenWithQVerify(Pass Abort) ok 16 - xfailDataDrivenWithQVerify(Pass Abort)
ok 17 - xfailDataDrivenWithQVerify(Pass Continue) ok 17 - xfailDataDrivenWithQVerify(Pass Continue)
not ok 18 - xfailDataDrivenWithQVerify(Fail Abort) # TODO This test should xfail not ok 18 - xfailDataDrivenWithQVerify(Fail Abort) # TODO This test should xfail
--- ---
at: tst_ExpectFail::xfailDataDrivenWithQVerify() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0) extensions:
file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp messages:
line: 0 - severity: xfail
message: This test should xfail
at: tst_ExpectFail::xfailDataDrivenWithQVerify() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0)
file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
line: 0
... ...
not ok 19 - xfailDataDrivenWithQVerify(Fail Continue) # TODO This test should xfail not ok 19 - xfailDataDrivenWithQVerify(Fail Continue) # TODO This test should xfail
--- ---
at: tst_ExpectFail::xfailDataDrivenWithQVerify() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0) extensions:
file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp messages:
line: 0 - severity: xfail
message: This test should xfail
at: tst_ExpectFail::xfailDataDrivenWithQVerify() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0)
file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
line: 0
... ...
ok 20 - xfailDataDrivenWithQCompare(Pass Abort) ok 20 - xfailDataDrivenWithQCompare(Pass Abort)
ok 21 - xfailDataDrivenWithQCompare(Pass Continue) ok 21 - xfailDataDrivenWithQCompare(Pass Continue)
not ok 22 - xfailDataDrivenWithQCompare(Fail Abort) # TODO This test should xfail not ok 22 - xfailDataDrivenWithQCompare(Fail Abort) # TODO This test should xfail
--- ---
at: tst_ExpectFail::xfailDataDrivenWithQCompare() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0) extensions:
file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp messages:
line: 0 - severity: xfail
message: This test should xfail
at: tst_ExpectFail::xfailDataDrivenWithQCompare() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0)
file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
line: 0
... ...
not ok 23 - xfailDataDrivenWithQCompare(Fail Continue) # TODO This test should xfail not ok 23 - xfailDataDrivenWithQCompare(Fail Continue) # TODO This test should xfail
--- ---
at: tst_ExpectFail::xfailDataDrivenWithQCompare() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0) extensions:
file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp messages:
line: 0 - severity: xfail
message: This test should xfail
at: tst_ExpectFail::xfailDataDrivenWithQCompare() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0)
file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
line: 0
... ...
ok 24 - xfailOnWrongRow(Fail Abort) ok 24 - xfailOnWrongRow(Fail Abort)
--- ---
@ -135,15 +190,23 @@ ok 25 - xfailOnWrongRow(Fail Continue)
... ...
not ok 26 - xfailOnAnyRow(Fail Abort) # TODO This test should xfail not ok 26 - xfailOnAnyRow(Fail Abort) # TODO This test should xfail
--- ---
at: tst_ExpectFail::xfailOnAnyRow() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0) extensions:
file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp messages:
line: 0 - severity: xfail
message: This test should xfail
at: tst_ExpectFail::xfailOnAnyRow() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0)
file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
line: 0
... ...
not ok 27 - xfailOnAnyRow(Fail Continue) # TODO This test should xfail not ok 27 - xfailOnAnyRow(Fail Continue) # TODO This test should xfail
--- ---
at: tst_ExpectFail::xfailOnAnyRow() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0) extensions:
file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp messages:
line: 0 - severity: xfail
message: This test should xfail
at: tst_ExpectFail::xfailOnAnyRow() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0)
file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
line: 0
... ...
not ok 28 - xfailWithoutCheck(Fail Abort) not ok 28 - xfailWithoutCheck(Fail Abort)
--- ---
@ -169,12 +232,7 @@ ok 33 - xpassContinue() # TODO 'true' returned TRUE unexpectedly. ()
ok 34 - xpassContinueSkip() # TODO 'true' returned TRUE unexpectedly. () ok 34 - xpassContinueSkip() # TODO 'true' returned TRUE unexpectedly. ()
ok 34 - xpassContinueSkip() # SKIP This should be reached but not increment skip-count ok 34 - xpassContinueSkip() # SKIP This should be reached but not increment skip-count
ok 35 - xpassContinueXfailAbort() # TODO 'true' returned TRUE unexpectedly. () ok 35 - xpassContinueXfailAbort() # TODO 'true' returned TRUE unexpectedly. ()
not ok 36 - xpassContinueXfailAbort() # TODO This test should xfail but not add to totals # xfail: This test should xfail but not add to totals
---
at: tst_ExpectFail::xpassContinueXfailAbort() (qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp:0)
file: qtbase/tests/auto/testlib/selftests/expectfail/tst_expectfail.cpp
line: 0
...
ok 36 - xpassAbortDataDrivenWithQVerify(XPass) # TODO 'true' returned TRUE unexpectedly. () ok 36 - xpassAbortDataDrivenWithQVerify(XPass) # TODO 'true' returned TRUE unexpectedly. ()
ok 37 - xpassAbortDataDrivenWithQVerify(Pass) ok 37 - xpassAbortDataDrivenWithQVerify(Pass)
ok 38 - xpassContinueDataDrivenWithQVerify(XPass) # TODO 'true' returned TRUE unexpectedly. () ok 38 - xpassContinueDataDrivenWithQVerify(XPass) # TODO 'true' returned TRUE unexpectedly. ()

23
tests/auto/testlib/selftests/expected_strcmp.tap
View File

@ -2,12 +2,6 @@ TAP version 13
# tst_StrCmp # tst_StrCmp
ok 1 - initTestCase() ok 1 - initTestCase()
ok 2 - compareCharStars() ok 2 - compareCharStars()
not ok 3 - compareByteArray() # TODO Next test should fail
---
at: tst_StrCmp::compareByteArray() (qtbase/tests/auto/testlib/selftests/strcmp/tst_strcmp.cpp:0)
file: qtbase/tests/auto/testlib/selftests/strcmp/tst_strcmp.cpp
line: 0
...
not ok 3 - compareByteArray() not ok 3 - compareByteArray()
--- ---
type: QCOMPARE type: QCOMPARE
@ -19,6 +13,23 @@ not ok 3 - compareByteArray()
at: tst_StrCmp::compareByteArray() (qtbase/tests/auto/testlib/selftests/strcmp/tst_strcmp.cpp:0) at: tst_StrCmp::compareByteArray() (qtbase/tests/auto/testlib/selftests/strcmp/tst_strcmp.cpp:0)
file: qtbase/tests/auto/testlib/selftests/strcmp/tst_strcmp.cpp file: qtbase/tests/auto/testlib/selftests/strcmp/tst_strcmp.cpp
line: 0 line: 0
extensions:
messages:
- severity: xfail
message: Next test should fail
at: tst_StrCmp::compareByteArray() (qtbase/tests/auto/testlib/selftests/strcmp/tst_strcmp.cpp:0)
file: qtbase/tests/auto/testlib/selftests/strcmp/tst_strcmp.cpp
line: 0
- severity: xfail
message: Next test should fail
at: tst_StrCmp::compareByteArray() (qtbase/tests/auto/testlib/selftests/strcmp/tst_strcmp.cpp:0)
file: qtbase/tests/auto/testlib/selftests/strcmp/tst_strcmp.cpp
line: 0
- severity: xfail
message: Next test should fail
at: tst_StrCmp::compareByteArray() (qtbase/tests/auto/testlib/selftests/strcmp/tst_strcmp.cpp:0)
file: qtbase/tests/auto/testlib/selftests/strcmp/tst_strcmp.cpp
line: 0
... ...
not ok 4 - failByteArray() not ok 4 - failByteArray()
--- ---