Run ES5 tests on Linux/x86-64

This requires including the test suite as a submodule by default and
bumping the module to a new sha1 that removes some particularly long
paths.

Also force the test suite to run under a US locale, as tests like
ch15/15.5/15.5.4/15.5.4.7/S15.5.4.7_A1_T11 cannot deal with locale
dependent output for date formatting.

The test harness now returns a non-zero exit code when a test fails.
This is used by the QTestLib wrapper to determine success or failure.

The tests with JIT are run, but the tests with the interpreter are
omitted at the moment until the last failure is fixed.

The tests add about 10-15 minutes extra time to the total time it takes
to run tests on Linux in the CI.

Change-Id: Id01fd3b41350f9c9a6ce9e43236f51f7f0fb71c8
Reviewed-by: Robin Burchell <robin.burchell@crimson.no>

.gitmodules

@@ -1,4 +1,3 @@
-[submodule "tests/manual/v4/test262"]
-	path = tests/manual/v4/test262
+[submodule "tests/auto/qml/ecmascripttests/test262"]
+	path = tests/auto/qml/ecmascripttests/test262
 	url = ../qtdeclarative-testsuites.git
-	update = none

tests/auto/qml/ecmascripttests/ecmascripttests.pro

@@ -0,0 +1,20 @@
+CONFIG += testcase
+TARGET = tst_ecmascripttests
+QT += testlib
+macos:CONFIG -= app_bundle
+SOURCES += tst_ecmascripttests.cpp
+DEFINES += SRCDIR=\\\"$$PWD\\\"
+
+TESTSCRIPT=$$PWD/test262.py
+isEmpty(V4CMD): V4CMD = qmljs
+
+checkjittarget.target = check-jit
+checkjittarget.commands = python $$TESTSCRIPT --command=$$V4CMD --parallel --with-test-expectations --update-expectations
+checkjittarget.depends = all
+QMAKE_EXTRA_TARGETS += checkjittarget
+
+checkmothtarget.target = check-interpreter
+checkmothtarget.commands = python $$TESTSCRIPT --command=\"$$V4CMD --interpret\" --parallel --with-test-expectations
+checkmothtarget.depends = all
+QMAKE_EXTRA_TARGETS += checkmothtarget
+

tests/auto/qml/ecmascripttests/test262

@@ -0,0 +1 @@
+Subproject commit d60c4ed97e69639bc5bc1db43a98828debf80c8a

tests/auto/qml/ecmascripttests/test262.py

@@ -555,6 +555,7 @@ class TestSuite(object):
     print
     if update_expectations:
         self.expectations.update(progress)
+    return progress.failed == 0
 
   def Print(self, tests):
     cases = self.EnumerateTests(tests)
@@ -567,6 +568,7 @@ def Main():
   # Uncomment the next line for more logging info.
   #logging.basicConfig(level=logging.DEBUG)
   os.environ["TZ"] = "PST8PDT"
+  os.environ["LANG"] = "en_US.UTF-8"
   parser = BuildOptions()
   (options, args) = parser.parse_args()
   ValidateOptions(options)
@@ -578,18 +580,21 @@ def Main():
   test_suite.Validate()
   if options.cat:
     test_suite.Print(args)
+    return 0
   else:
-    test_suite.Run(options.command, args,
+    if test_suite.Run(options.command, args,
                       options.summary or options.full_summary,
                       options.full_summary,
                       options.parallel,
-                   options.update_expectations)
+                      options.update_expectations):
+      return 0
+    else:
+      return 1
 
 
 if __name__ == '__main__':
   try:
-    Main()
-    sys.exit(0)
+    sys.exit(Main())
   except Test262Error, e:
     print "Error: %s" % e.message
     sys.exit(1)

tests/auto/qml/ecmascripttests/tst_ecmascripttests.cpp

@@ -0,0 +1,77 @@
+/****************************************************************************
+**
+** Copyright (C) 2017 The Qt Company Ltd.
+** Contact: https://www.qt.io/licensing/
+**
+** This file is part of the test suite of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:GPL-EXCEPT$
+** Commercial License Usage
+** Licensees holding valid commercial Qt licenses may use this file in
+** accordance with the commercial license agreement provided with the
+** Software or, alternatively, in accordance with the terms contained in
+** a written agreement between you and The Qt Company. For licensing terms
+** and conditions see https://www.qt.io/terms-conditions. For further
+** information use the contact form at https://www.qt.io/contact-us.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3 as published by the Free Software
+** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+** included in the packaging of this file. Please review the following
+** information to ensure the GNU General Public License requirements will
+** be met: https://www.gnu.org/licenses/gpl-3.0.html.
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+
+
+#include <QtTest/QtTest>
+#include <QProcess>
+#include <QLibraryInfo>
+
+class tst_EcmaScriptTests : public QObject
+{
+    Q_OBJECT
+private slots:
+    void runTests_data();
+    void runTests();
+};
+
+void tst_EcmaScriptTests::runTests_data()
+{
+    QTest::addColumn<QString>("qmljsParameter");
+
+    QTest::newRow("jit") << QStringLiteral("--jit");
+    // Not passing yet: QTest::newRow("interpreter") << QStringLiteral("--interpret");
+}
+
+void tst_EcmaScriptTests::runTests()
+{
+#if defined(Q_OS_LINUX) && defined(Q_PROCESSOR_X86_64)
+    QFETCH(QString, qmljsParameter);
+
+    QProcess process;
+    process.setProcessChannelMode(QProcess::ForwardedChannels);
+    process.setWorkingDirectory(QLatin1String(SRCDIR));
+    process.setProgram("python");
+    process.setArguments(QStringList() << "test262.py" << "--command=" + QLibraryInfo::location(QLibraryInfo::BinariesPath) + "/qmljs " + qmljsParameter << "--parallel" << "--with-test-expectations");
+
+    qDebug() << "Going to run" << process.program() << process.arguments() << "in" << process.workingDirectory();
+
+    process.start();
+    QVERIFY(process.waitForStarted());
+    const int timeoutInMSecs = 20 * 60 * 1000;
+    QVERIFY2(process.waitForFinished(timeoutInMSecs), "Tests did not terminate in time -- see output above for details");
+    QVERIFY2(process.exitStatus() == QProcess::NormalExit, "Running the test harness failed -- see output above for details");
+    QVERIFY2(process.exitCode() == 0, "Tests failed -- see output above for details");
+#else
+    QSKIP("Currently the ecmascript tests are only run on Linux/x86-64");
+#endif
+}
+
+QTEST_MAIN(tst_EcmaScriptTests)
+
+#include "tst_ecmascripttests.moc"
+

tests/auto/qml/qml.pro

@@ -62,7 +62,8 @@ PRIVATETESTS += \
     qqmlimport \
     qqmlobjectmodel \
     qmldiskcache \
-    qv4mm
+    qv4mm \
+    ecmascripttests
 
 qtHaveModule(widgets) {
     PUBLICTESTS += \

tests/auto/qml/qmlmin/tst_qmlmin.cpp

@@ -84,6 +84,7 @@ void tst_qmlmin::initTestCase()
     excludedDirs << "doc/src/snippets/qtquick1/qtbinding";
     excludedDirs << "doc/src/snippets/qtquick1/imports";
     excludedDirs << "tests/manual/v4";
+    excludedDirs << "tests/auto/qml/ecmascripttests";
     excludedDirs << "tests/auto/qml/qmllint";
 
     // Add invalid files (i.e. files with syntax errors)

tests/manual/v4/test262

@@ -1 +0,0 @@
-Subproject commit 9741ac4655808ac46c127e3d1d8ba3d27ada618e

tests/manual/v4/tests.pro

@@ -1,15 +0,0 @@
-TEMPLATE = aux
-
-TESTSCRIPT=$$PWD/test262.py
-isEmpty(V4CMD): V4CMD = qmljs
-
-checktarget.target = check
-checktarget.commands = python $$TESTSCRIPT --command=$$V4CMD --parallel --with-test-expectations --update-expectations
-checktarget.depends = all
-QMAKE_EXTRA_TARGETS += checktarget
-
-checkmothtarget.target = check-interpreter
-checkmothtarget.commands = python $$TESTSCRIPT --command=\"$$V4CMD --interpret\" --parallel --with-test-expectations
-checkmothtarget.depends = all
-QMAKE_EXTRA_TARGETS += checkmothtarget
-