QmlTest: Prevent interleaved execution of TestCases

Run all TestCase instances on a timer in a singleton. This way, even if
one TestCase manages to trigger a different one while it's still
running, the new TestCase won't be executed until the old one is
finished.

Fixes: QTBUG-98350
Change-Id: I1797b5487f2c70fd2edfdbf8cf4c43a6353b12c8
Reviewed-by: Tor Arne Vestbø <tor.arne.vestbo@qt.io>

src/qmltest/CMakeLists.txt

@@ -9,6 +9,10 @@ set_source_files_properties(testlogger.js PROPERTIES
     QT_QML_SKIP_QMLDIR_ENTRY TRUE
 )
 
+set_source_files_properties(TestSchedule.qml PROPERTIES
+    QT_QML_SINGLETON_TYPE TRUE
+)
+
 qt_internal_add_qml_module(QuickTest
     URI "QtTest"
     VERSION "${PROJECT_VERSION}"
@@ -25,8 +29,9 @@ qt_internal_add_qml_module(QuickTest
         quicktestresult.cpp quicktestresult_p.h
         quicktestutil.cpp quicktestutil_p.h
     QML_FILES
-        TestCase.qml
         SignalSpy.qml
+        TestCase.qml
+        TestSchedule.qml
         testlogger.js
     DEFINES
         QT_NO_FOREACH

src/qmltest/TestCase.qml

@@ -1849,7 +1849,11 @@ Item {
 
     /*! \internal */
     function qtest_run() {
-        if (TestLogger.log_start_test()) {
+        if (!when || completed || running || !qtest_componentCompleted)
+            return;
+
+        verify(TestLogger.log_can_start_test(qtest_testId))
+        if (TestLogger.log_start_test(qtest_testId)) {
             qtest_results.reset()
             qtest_results.testCaseName = name
             qtest_results.startLogging()
@@ -2020,8 +2024,8 @@ Item {
     onWhenChanged: {
         if (when != qtest_prevWhen) {
             qtest_prevWhen = when
-            if (when && !completed && !running && qtest_componentCompleted)
-                qtest_run()
+            if (when)
+                TestSchedule.testCases.push(testCase)
         }
     }
 
@@ -2041,7 +2045,7 @@ Item {
         if (optional)
             TestLogger.log_optional_test(qtest_testId)
         qtest_prevWhen = when
-        if (when && !completed && !running)
-            qtest_run()
+        if (when)
+            TestSchedule.testCases.push(testCase)
     }
 }

src/qmltest/TestSchedule.qml

@@ -0,0 +1,29 @@
+// Copyright (C) 2022 The Qt Company Ltd.
+// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
+
+pragma Singleton
+import QtQml
+
+Timer {
+    property list<QtObject> testCases
+    property QtObject currentTest: null
+
+    running: testCases.length > 0 && !currentTest
+    interval: 1
+    repeat: true
+
+    onTriggered: {
+        if (currentTest) {
+            console.error("Interleaved test execution detected. This shouldn't happen")
+            return;
+        }
+
+        try {
+            currentTest = testCases.shift()
+            currentTest.qtest_run()
+        } finally {
+            currentTest = null
+        }
+    }
+
+}

src/qmltest/testlogger.js

@@ -9,7 +9,7 @@ function log_init_results()
 {
     if (!testResults) {
         testResults = {
-            reportedStart: false,
+            runningTest: -1,
             nextId: 0,
             testCases: []
         }
@@ -36,16 +36,21 @@ function log_mandatory_test(testId)
 {
     log_init_results()
     var index = testResults.testCases.indexOf(testId)
-    if (index == -1)
+    if (index === -1)
         testResults.testCases.push(testId)
 }
 
-function log_start_test()
+function log_can_start_test(testId)
+{
+    return !testResults || testResults.runningTest === -1 || testResults.runningTest === testId;
+}
+
+function log_start_test(testId)
 {
     log_init_results()
-    if (testResults.reportedStart)
+    if (testResults.runningTest === testId)
         return false
-    testResults.reportedStart = true
+    testResults.runningTest = testId
     return true
 }
 
@@ -54,5 +59,6 @@ function log_complete_test(testId)
     var index = testResults.testCases.indexOf(testId)
     if (index >= 0)
         testResults.testCases.splice(index, 1)
+    testResults.runningTest = -1
     return testResults.testCases.length > 0
 }

tests/auto/qmltest/selftests/tst_multiTestCase.qml

@@ -0,0 +1,117 @@
+// Copyright (C) 2022 The Qt Company Ltd.
+// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0
+
+import QtTest
+import QtQuick
+
+Item {
+    id: root
+    property int finished: 0
+    property TestCase currentTest
+
+    Repeater {
+        model: 100
+        TestCase {
+            id: test
+
+            name: "A"
+
+            required property int index
+            when: root.children.length > 50 && ((root.finished + index) % 17 == 0 || cutoff.completed)
+
+            function initTestCase() {
+                compare(root.currentTest, null)
+                root.currentTest = test
+            }
+
+            function init() {
+                compare(root.currentTest, test)
+            }
+
+            function test_do() {
+                compare(root.currentTest, test)
+                ++root.finished
+                wait(index / 10)
+                compare(root.currentTest, test)
+            }
+
+            function cleanup() {
+                compare(root.currentTest, test)
+            }
+
+            function cleanupTestCase() {
+                compare(root.currentTest, test)
+                root.currentTest = null
+            }
+        }
+    }
+
+    TestCase {
+        id: cutoff
+        name: "C"
+        when: root.finished > 80
+
+        function initTestCase() {
+            compare(root.currentTest, null)
+            root.currentTest = cutoff
+        }
+
+        function init() {
+            compare(root.currentTest, cutoff)
+        }
+
+        function test_cutoff() {
+            compare(root.currentTest, cutoff)
+        }
+
+        function cleanup() {
+            compare(root.currentTest, cutoff)
+        }
+
+        function cleanupTestCase() {
+            compare(root.currentTest, cutoff)
+            root.currentTest = null
+        }
+    }
+
+    TestCase {
+        id: sum
+        name: "B"
+        when: root.finished === 100
+
+        function initTestCase() {
+            compare(root.currentTest, null)
+            root.currentTest = sum
+        }
+
+        function init() {
+            compare(root.currentTest, sum)
+        }
+
+
+        function test_sum() {
+            compare(root.currentTest, sum)
+            var numTests = 0;
+            for (var i in root.children) {
+                var test = root.children[i];
+                if (test.name === "A") {
+                    verify(test.completed)
+                    ++numTests
+                }
+            }
+
+            compare(numTests, 100)
+            compare(root.currentTest, sum)
+        }
+
+
+        function cleanup() {
+            compare(root.currentTest, sum)
+        }
+
+        function cleanupTestCase() {
+            compare(root.currentTest, sum)
+            root.currentTest = null
+        }
+    }
+}