Qt Quick Test: support tags filter on tests run

Implemented logic for filtering tests with 'case::function:tag' run syntax.

Fixes: QTBUG-71645
Change-Id: I9243636b098b4e9ccb4c74d08679bbbb9f3e8a65
Reviewed-by: Mitch Curtis <mitch.curtis@qt.io>

src/imports/testlib/TestCase.qml

@@ -1787,17 +1787,28 @@ Item {
         running = true
 
         // Check the run list to see if this class is mentioned.
-        var functionsToRun = qtest_results.functionsToRun
+        let checkNames = false
-        if (functionsToRun.length > 0) {
+        let testsToRun = {} // explicitly provided function names to run and their tags for data-driven tests
+
+        if (qtest_results.functionsToRun.length > 0) {
+            checkNames = true
             var found = false
-            var list = []
+
             if (name.length > 0) {
-                var prefix = name + "::"
+                for (var index in qtest_results.functionsToRun) {
-                for (var index in functionsToRun) {
+                    let caseFuncName = qtest_results.functionsToRun[index]
-                    if (functionsToRun[index].indexOf(prefix) == 0) {
+                    if (caseFuncName.indexOf(name + "::") != 0)
-                        list.push(functionsToRun[index])
+                        continue
-                        found = true
+
-                    }
+                    found = true
+                    let funcName = caseFuncName.substring(name.length + 2)
+
+                    if (!(funcName in testsToRun))
+                        testsToRun[funcName] = []
+
+                    let tagName = qtest_results.tagsToRun[index]
+                    if (tagName.length > 0) // empty tags mean run all rows
+                        testsToRun[funcName].push(tagName)
                 }
             }
             if (!found) {
@@ -1809,7 +1820,6 @@ Item {
                 qtest_results.testCaseName = ""
                 return
             }
-            functionsToRun = list
         }
 
         // Run the initTestCase function.
@@ -1834,17 +1844,15 @@ Item {
             }
             testList.sort()
         }
-        var checkNames = (functionsToRun.length > 0)
+
         for (var index in testList) {
             var prop = testList[index]
+
+            if (checkNames && !(prop in testsToRun))
+                continue
+
             var datafunc = prop + "_data"
             var isBenchmark = (prop.indexOf("benchmark_") == 0)
-            if (checkNames) {
-                var index = functionsToRun.indexOf(name + "::" + prop)
-                if (index < 0)
-                    continue
-                functionsToRun.splice(index, 1)
-            }
             qtest_results.functionName = prop
 
             if (!(datafunc in testCase))
@@ -1854,12 +1862,22 @@ Item {
                 if (qtest_runInternal(datafunc)) {
                     var table = qtest_testCaseResult
                     var haveData = false
+
+                    let checkTags = (checkNames && testsToRun[prop].length > 0)
+
                     qtest_results.initTestTable()
                     for (var index in table) {
                         haveData = true
                         var row = table[index]
                         if (!row.tag)
                             row.tag = "row " + index    // Must have something
+                        if (checkTags) {
+                            let tags = testsToRun[prop]
+                            let tagIdx = tags.indexOf(row.tag)
+                            if (tagIdx < 0)
+                                continue
+                            tags.splice(tagIdx, 1)
+                        }
                         qtest_results.dataTag = row.tag
                         if (isBenchmark)
                             qtest_runBenchmarkFunction(prop, row)
@@ -1884,6 +1902,9 @@ Item {
             }
             qtest_results.finishTestFunction()
             qtest_results.skipped = false
+
+            if (checkNames && testsToRun[prop].length <= 0)
+                delete testsToRun[prop]
         }
 
         // Run the cleanupTestCase function.
@@ -1892,8 +1913,21 @@ Item {
         qtest_runInternal("cleanupTestCase")
 
         // Complain about missing functions that we were supposed to run.
-        if (functionsToRun.length > 0)
+        if (checkNames) {
-            qtest_results.fail("Could not find functions: " + functionsToRun, "", 0)
+            let missingTests = []
+            for (var func in testsToRun) {
+                let caseFuncName = name + '::' + func
+                let tags = testsToRun[func]
+                if (tags.length <= 0)
+                    missingTests.push(caseFuncName)
+                else
+                    for (var i in tags)
+                        missingTests.push(caseFuncName + ':' + tags[i])
+            }
+            missingTests.sort()
+            if (missingTests.length > 0)
+                qtest_results.fail("Could not find test functions: " + missingTests, "", 0)
+        }
 
         // Clean up and exit.
         running = false

src/imports/testlib/plugins.qmltypes

@@ -195,6 +195,7 @@ Module {
         Property { name: "failCount"; type: "int"; isReadonly: true }
         Property { name: "skipCount"; type: "int"; isReadonly: true }
         Property { name: "functionsToRun"; type: "QStringList"; isReadonly: true }
+        Property { name: "tagsToRun"; type: "QStringList"; isReadonly: true }
         Signal { name: "programNameChanged" }
         Method { name: "reset" }
         Method { name: "startLogging" }

src/qmltest/quicktestresult.cpp

@@ -380,6 +380,16 @@ QStringList QuickTestResult::functionsToRun() const
     return QTest::testFunctions;
 }
 
+/*!
+    \qmlproperty list<string> TestResult::tagsToRun
+
+    This property returns the list of test function's data tags to be run
+*/
+QStringList QuickTestResult::tagsToRun() const
+{
+    return QTest::testTags;
+}
+
 /*!
     \qmlmethod TestResult::reset()
 

src/qmltest/quicktestresult_p.h

@@ -76,6 +76,7 @@ class Q_QUICK_TEST_EXPORT QuickTestResult : public QObject
     Q_PROPERTY(int failCount READ failCount)
     Q_PROPERTY(int skipCount READ skipCount)
     Q_PROPERTY(QStringList functionsToRun READ functionsToRun)
+    Q_PROPERTY(QStringList tagsToRun READ tagsToRun)
 public:
     QuickTestResult(QObject *parent = nullptr);
     ~QuickTestResult() override;
@@ -107,6 +108,7 @@ public:
     int skipCount() const;
 
     QStringList functionsToRun() const;
+    QStringList tagsToRun() const;
 
 public Q_SLOTS:
     void reset();

tests/auto/quicktest/testfiltering/tst_testfiltering.cpp

@@ -38,6 +38,10 @@ private slots:
     void twoFilters();
     void twoFiltersWithOneMatch();
     void manyFilters();
+    void filterTestWithDefaultDataTags();
+    void filterTestWithDataTags();
+    void filterTestByDataTag();
+    void filterInvalidDataTag();
 };
 
 
@@ -131,6 +135,62 @@ void tst_TestFiltering::manyFilters()
     QCOMPARE(process.exitCode(), 0);
 }
 
+void tst_TestFiltering::filterTestWithDefaultDataTags()
+{
+    QProcess process;
+    process.start(testExe, { QLatin1String("Third::test_default_tags"), });
+
+    QVERIFY(process.waitForFinished());
+
+    const QString output = process.readAll();
+    QVERIFY(output.contains(QLatin1String("Totals: 5 passed")));
+    QVERIFY(output.contains(QLatin1String(" 2 skipped")));
+    QCOMPARE(process.exitStatus(), QProcess::NormalExit);
+    QCOMPARE(process.exitCode(), 0);
+}
+
+void tst_TestFiltering::filterTestWithDataTags()
+{
+    QProcess process;
+    process.start(testExe, { QLatin1String("Third::test_tags"), });
+
+    QVERIFY(process.waitForFinished());
+
+    const QString output = process.readAll();
+    QVERIFY(output.contains(QLatin1String("Totals: 4 passed")));
+    QVERIFY(output.contains(QLatin1String(" 1 skipped")));
+    QCOMPARE(process.exitStatus(), QProcess::NormalExit);
+    QCOMPARE(process.exitCode(), 0);
+}
+
+void tst_TestFiltering::filterTestByDataTag()
+{
+    QProcess process;
+    process.start(testExe, { QLatin1String("Third::test_default_tags:init_2"),
+                             QLatin1String("Third::test_default_tags:skip_3"),
+                             QLatin1String("Third::test_tags:baz"),
+                             QLatin1String("Third::test_tags:bar"), });
+
+    QVERIFY(process.waitForFinished());
+
+    const QString output = process.readAll();
+    QVERIFY(output.contains(QLatin1String("Totals: 4 passed")));
+    QVERIFY(output.contains(QLatin1String(" 2 skipped")));
+    QCOMPARE(process.exitStatus(), QProcess::NormalExit);
+    QCOMPARE(process.exitCode(), 0);
+}
+
+void tst_TestFiltering::filterInvalidDataTag()
+{
+    QProcess process;
+    process.start(testExe, { QLatin1String("Third::test_tags:invalid_tag") });
+
+    QVERIFY(process.waitForFinished());
+
+    QCOMPARE(process.exitStatus(), QProcess::NormalExit);
+    QCOMPARE(process.exitCode(), 1);
+}
+
 QTEST_MAIN(tst_TestFiltering);
 
 #include "tst_testfiltering.moc"