qiurui
/
qtdeclarative
mirror of https://github.com/qt/qtdeclarative.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Compiler: Record statistics about aot compilation 

This patch introduces the collection of statistics about the
ahead-of-time compilation of functions and bindings to Cpp by
qmlcachegen. This is done by having qmlcachegen save an aotstats file
for every qml file it compiles. This file contains, for every function
and binding, whether the Cpp codegen was successful, its duration and a
potential error message

Task-number: QTBUG-124667
Change-Id: Iba9a72be04f6642688533a3ae12ea687296c85e1
Reviewed-by: Fabian Kosmale <fabian.kosmale@qt.io>
This commit is contained in:
Olivier De Cannière 2024-05-24 13:21:44 +02:00
parent b803c5baaf
commit 8827af0fe8
13 changed files with 447 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
src/qml/Qt6QmlMacros.cmake
View File

@ -2395,6 +2395,8 @@ function(qt6_target_qml_sources target)
"$<${have_direct_calls}:--direct-calls>"
"$<${have_arguments}:${arguments}>"
${qrc_resource_args}
"--dump-aot-stats"
"--module-id=${arg_URI}(${target})"
)
# For direct evaluation in if() below
@ -2641,9 +2643,16 @@ function(qt6_target_qml_sources target)
set(qmlcachegen_cmd "${qmlcachegen}")
endif()
set(aotstats_file "")
if("${qml_file_src}" MATCHES ".+\\.qml")
set(aotstats_file "${compiled_file}.aotstats")
endif()
_qt_internal_get_tool_wrapper_script_path(tool_wrapper)
add_custom_command(
OUTPUT ${compiled_file}
OUTPUT
${compiled_file}
${aotstats_file}
COMMAND ${CMAKE_COMMAND} -E make_directory ${out_dir}
COMMAND
${tool_wrapper}

1
src/qmlcompiler/CMakeLists.txt
View File

@ -16,6 +16,7 @@ qt_internal_add_module(QmlCompiler
qqmljscodegenerator.cpp qqmljscodegenerator_p.h
qqmljscompilepass_p.h
qqmljscompiler.cpp qqmljscompiler_p.h
qqmljscompilerstats.cpp qqmljscompilerstats_p.h
qqmljsfunctioninitializer.cpp qqmljsfunctioninitializer_p.h
qqmljsimporter.cpp qqmljsimporter_p.h
qqmljsimportvisitor.cpp qqmljsimportvisitor_p.h

29
src/qmlcompiler/qqmljscompiler.cpp
View File

@ -6,6 +6,7 @@
#include <private/qqmlirbuilder_p.h>
#include <private/qqmljsbasicblocks_p.h>
#include <private/qqmljscodegenerator_p.h>
#include <private/qqmljscompilerstats_p.h>
#include <private/qqmljsfunctioninitializer_p.h>
#include <private/qqmljsimportvisitor_p.h>
#include <private/qqmljslexer_p.h>
@ -679,7 +680,8 @@ std::variant<QQmlJSAotFunction, QQmlJS::DiagnosticMessage> QQmlJSAotCompiler::co
const QString name = m_document->stringAt(irBinding.propertyNameIndex);
QQmlJSCompilePass::Function function = initializer.run(
context, name, astNode, irBinding, &error);
const QQmlJSAotFunction aotFunction = doCompile(context, &function, &error);
const QQmlJSAotFunction aotFunction = doCompileAndRecordAotStats(
context, &function, &error, name, astNode->firstSourceLocation());
if (error.isValid()) {
// If it's a signal and the function just returns a closure, it's harmless.
@ -703,7 +705,8 @@ std::variant<QQmlJSAotFunction, QQmlJS::DiagnosticMessage> QQmlJSAotCompiler::co
&m_typeResolver, m_currentObject->location, m_currentScope->location);
QQmlJS::DiagnosticMessage error;
QQmlJSCompilePass::Function function = initializer.run(context, name, astNode, &error);
const QQmlJSAotFunction aotFunction = doCompile(context, &function, &error);
const QQmlJSAotFunction aotFunction = doCompileAndRecordAotStats(
context, &function, &error, name, astNode->firstSourceLocation());
if (error.isValid())
return diagnose(error.message, QtWarningMsg, error.loc);
@ -783,4 +786,26 @@ QQmlJSAotFunction QQmlJSAotCompiler::doCompile(
return error->isValid() ? compileError() : result;
}
QQmlJSAotFunction QQmlJSAotCompiler::doCompileAndRecordAotStats(
const QV4::Compiler::Context *context, QQmlJSCompilePass::Function *function,
QQmlJS::DiagnosticMessage *error, const QString &name, QQmlJS::SourceLocation location)
{
auto t1 = std::chrono::high_resolution_clock::now();
auto &&result = doCompile(context, function, error);
auto t2 = std::chrono::high_resolution_clock::now();
if (QQmlJS::QQmlJSAotCompilerStats::recordAotStats()) {
QQmlJS::AotStatsEntry entry;
entry.codegenDuration = std::chrono::duration_cast<std::chrono::microseconds>(t2 - t1);
entry.functionName = name;
entry.errorMessage = error->message;
entry.line = location.startLine;
entry.column = location.startColumn;
entry.codegenSuccessful = !error->isValid();
QQmlJS::QQmlJSAotCompilerStats::addEntry(function->qmlScope->filePath(), entry);
}
return result;
}
QT_END_NAMESPACE

12
src/qmlcompiler/qqmljscompiler_p.h
View File

@ -22,6 +22,7 @@
#include <private/qqmlirbuilder_p.h>
#include <private/qqmljscompilepass_p.h>
#include <private/qqmljscompilerstats_p.h>
#include <private/qqmljsdiagnosticmessage_p.h>
#include <private/qqmljsimporter_p.h>
#include <private/qqmljslogger_p.h>
@ -97,9 +98,14 @@ protected:
QQmlJSLogger *m_logger = nullptr;
private:
QQmlJSAotFunction doCompile(
const QV4::Compiler::Context *context, QQmlJSCompilePass::Function *function,
QQmlJS::DiagnosticMessage *error);
QQmlJSAotFunction doCompile(const QV4::Compiler::Context *context,
QQmlJSCompilePass::Function *function,
QQmlJS::DiagnosticMessage *error);
QQmlJSAotFunction doCompileAndRecordAotStats(const QV4::Compiler::Context *context,
QQmlJSCompilePass::Function *function,
QQmlJS::DiagnosticMessage *error,
const QString &name,
QQmlJS::SourceLocation location);
};
Q_DECLARE_OPERATORS_FOR_FLAGS(QQmlJSAotCompiler::Flags);

134
src/qmlcompiler/qqmljscompilerstats.cpp Normal file
View File

@ -0,0 +1,134 @@
// Copyright (C) 2024 The Qt Company Ltd.
// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0
#include "qqmljscompilerstats_p.h"
#include <QFile>
#include <QJsonArray>
#include <QJsonDocument>
#include <QJsonObject>
#include <QTextStream>
QT_BEGIN_NAMESPACE
namespace QQmlJS {
using namespace Qt::StringLiterals;
std::unique_ptr<AotStats> QQmlJSAotCompilerStats::s_instance = std::make_unique<AotStats>();
QString QQmlJSAotCompilerStats::s_moduleId;
bool QQmlJSAotCompilerStats::s_recordAotStats = false;
bool QQmlJS::AotStatsEntry::operator<(const AotStatsEntry &other) const
{
if (line == other.line)
return column < other.column;
return line < other.line;
}
AotStats AotStats::fromJsonDocument(const QJsonDocument &document)
{
QJsonArray modulesArray = document.array();
QQmlJS::AotStats result;
for (const auto &modulesArrayEntry : modulesArray) {
const auto &moduleObject = modulesArrayEntry.toObject();
QString moduleId = moduleObject[u"moduleId"_s].toString();
const QJsonArray &filesArray = moduleObject[u"moduleFiles"_s].toArray();
QHash<QString, QList<AotStatsEntry>> files;
for (const auto &filesArrayEntry : filesArray) {
const QJsonObject &fileObject = filesArrayEntry.toObject();
QString filepath = fileObject[u"filepath"_s].toString();
const QJsonArray &statsArray = fileObject[u"entries"_s].toArray();
QList<AotStatsEntry> stats;
for (const auto &statsArrayEntry : statsArray) {
const auto &statsObject = statsArrayEntry.toObject();
QQmlJS::AotStatsEntry stat;
auto micros = statsObject[u"durationMicroseconds"_s].toInteger();
stat.codegenDuration = std::chrono::microseconds(micros);
stat.functionName = statsObject[u"functionName"_s].toString();
stat.errorMessage = statsObject[u"errorMessage"_s].toString();
stat.line = statsObject[u"line"_s].toInt();
stat.column = statsObject[u"column"_s].toInt();
stat.codegenSuccessful = statsObject[u"codegenSuccessfull"_s].toBool();
stats.append(std::move(stat));
}
std::sort(stats.begin(), stats.end());
files[filepath] = stats;
}
result.m_entries[moduleId] = files;
}
return result;
}
QJsonDocument AotStats::toJsonDocument() const
{
QJsonArray modulesArray;
for (auto it1 = m_entries.begin(); it1 != m_entries.end(); ++it1) {
const QString moduleId = it1.key();
const QHash<QString, QList<AotStatsEntry>> &files = it1.value();
QJsonArray filesArray;
for (auto it2 = files.begin(); it2 != files.end(); ++it2) {
const QString &filename = it2.key();
const QList<AotStatsEntry> &stats = it2.value();
QJsonArray statsArray;
for (const auto &stat : stats) {
QJsonObject statObject;
auto micros = static_cast<qint64>(stat.codegenDuration.count());
statObject.insert(u"durationMicroseconds", micros);
statObject.insert(u"functionName", stat.functionName);
statObject.insert(u"errorMessage", stat.errorMessage);
statObject.insert(u"line", stat.line);
statObject.insert(u"column", stat.column);
statObject.insert(u"codegenSuccessfull", stat.codegenSuccessful);
statsArray.append(statObject);
}
QJsonObject o;
o.insert(u"filepath"_s, filename);
o.insert(u"entries"_s, statsArray);
filesArray.append(o);
}
QJsonObject o;
o.insert(u"moduleId"_s, moduleId);
o.insert(u"moduleFiles"_s, filesArray);
modulesArray.append(o);
}
return QJsonDocument(modulesArray);
}
void AotStats::addEntry(const QString &moduleId, const QString &filepath, AotStatsEntry entry)
{
m_entries[moduleId][filepath].append(entry);
}
bool AotStats::saveToDisk(const QString &filepath) const
{
QFile file(filepath);
if (!file.open(QIODevice::WriteOnly | QIODevice::Text | QIODevice::Truncate)) {
qDebug().noquote() << u"Could not open \"%1\""_s.arg(filepath);
return false;
}
file.write(this->toJsonDocument().toJson(QJsonDocument::Indented));
return true;
}
void QQmlJSAotCompilerStats::addEntry(QString filepath, QQmlJS::AotStatsEntry entry)
{
auto *aotstats = QQmlJSAotCompilerStats::instance();
aotstats->addEntry(s_moduleId, filepath, entry);
}
} // namespace QQmlJS
QT_END_NAMESPACE

88
src/qmlcompiler/qqmljscompilerstats_p.h Normal file
View File

@ -0,0 +1,88 @@
// Copyright (C) 2024 The Qt Company Ltd.
// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0
#ifndef QQMLJSCOMPILERSTATS_P_H
#define QQMLJSCOMPILERSTATS_P_H
//
// W A R N I N G
// -------------
//
// This file is not part of the Qt API. It exists purely as an
// implementation detail. This header file may change from version to
// version without notice, or even be removed.
//
// We mean it.
#include <qtqmlcompilerexports.h>
#include <QHash>
#include <QJsonDocument>
#include <private/qqmljsdiagnosticmessage_p.h>
#include <private/qqmljssourcelocation_p.h>
#include <memory>
QT_BEGIN_NAMESPACE
namespace QQmlJS {
struct Q_QMLCOMPILER_EXPORT AotStatsEntry
{
std::chrono::microseconds codegenDuration;
QString functionName;
QString errorMessage;
int line = 0;
int column = 0;
bool codegenSuccessful = true;
bool operator<(const AotStatsEntry &) const;
};
class Q_QMLCOMPILER_EXPORT AotStats
{
friend class QQmlJSAotCompilerStats;
public:
const QHash<QString, QHash<QString, QList<AotStatsEntry>>> &entries() const
{
return m_entries;
}
void addEntry(const QString &moduleId, const QString &filepath, AotStatsEntry entry);
bool saveToDisk(const QString &filepath) const;
static AotStats fromJsonDocument(const QJsonDocument &);
QJsonDocument toJsonDocument() const;
private:
// module Id -> filename -> stats m_entries
QHash<QString, QHash<QString, QList<AotStatsEntry>>> m_entries;
};
class Q_QMLCOMPILER_EXPORT QQmlJSAotCompilerStats
{
public:
static AotStats *instance() { return s_instance.get(); }
static bool recordAotStats() { return s_recordAotStats; }
static void setRecordAotStats(bool recordAotStats) { s_recordAotStats = recordAotStats; }
static const QString &moduleId() { return s_moduleId; }
static void setModuleId(QString moduleId) { s_moduleId = moduleId; }
static void addEntry(QString filepath, QQmlJS::AotStatsEntry entry);
private:
static std::unique_ptr<AotStats> s_instance;
static QString s_moduleId;
static bool s_recordAotStats;
};
} // namespace QQmlJS
QT_END_NAMESPACE
#endif // QQMLJSCOMPILERSTATS_P_H

1
tests/auto/qml/qmlcachegen/CMakeLists.txt
View File

@ -28,6 +28,7 @@ qt_internal_add_test(tst_qmlcachegen
Qt::Gui
Qt::QmlPrivate
Qt::QuickTestUtilsPrivate
Qt::QmlCompilerPrivate
TESTDATA ${test_data}
)

11
tests/auto/qml/qmlcachegen/data/aotstats/AotstatsClean.qml Normal file
View File

@ -0,0 +1,11 @@
import QtQml
QtObject {
property int i: 100
property int j: i * 2
function s() : string {
let s = "abc"
return s + "def "
}
}

7
tests/auto/qml/qmlcachegen/data/aotstats/AotstatsMixed.qml Normal file
View File

@ -0,0 +1,7 @@
import QtQml
QtObject {
property int i: Math.max(1, 2) // OK
function f() { return 1 } // Error: No specified return type
property string s: g() // Error: g?
}

5
tests/auto/qml/qmlcachegen/data/aotstats/cachegentest.qrc Normal file
View File

@ -0,0 +1,5 @@
<RCC>
<qresource prefix="/cachegentest">
<file alias="qmldir">qmldir</file>
</qresource>
</RCC>

3
tests/auto/qml/qmlcachegen/data/aotstats/qmldir Normal file
View File

@ -0,0 +1,3 @@
module cachegentest
AotstatsClean 254.0 AotstatsClean.qml
AotstatsMixed 254.0 AotstatsMixed.qml

133
tests/auto/qml/qmlcachegen/tst_qmlcachegen.cpp
View File

@ -3,6 +3,7 @@
#include <qtest.h>
#include <QJsonDocument>
#include <QQmlComponent>
#include <QQmlEngine>
#include <QProcess>
@ -11,6 +12,7 @@
#include <QSysInfo>
#include <QLoggingCategory>
#include <private/qqmlcomponent_p.h>
#include <private/qqmljscompilerstats_p.h>
#include <private/qqmlscriptdata_p.h>
#include <private/qv4compileddata_p.h>
#include <qtranslator.h>
@ -67,6 +69,10 @@ private slots:
void scriptStringCachegenInteraction();
void saveableUnitPointer();
void aotstatsSerialization();
void aotstatsGeneration_data();
void aotstatsGeneration();
};
// A wrapper around QQmlComponent to ensure the temporary reference counts
@ -895,6 +901,133 @@ void tst_qmlcachegen::saveableUnitPointer()
QCOMPARE(unit.flags, flags);
}
void tst_qmlcachegen::aotstatsSerialization()
{
const auto createEntry = [](const auto &d, const auto &n, const auto &e, const auto &l,
const auto &c, const auto &s) -> QQmlJS::AotStatsEntry {
QQmlJS::AotStatsEntry entry;
entry.codegenDuration = d;
entry.functionName = n;
entry.errorMessage = e;
entry.line = l;
entry.column = c;
entry.codegenSuccessful = s;
return entry;
};
const auto equal = [](const auto &e1, const auto &e2) -> bool {
return e1.codegenDuration == e2.codegenDuration && e1.functionName == e2.functionName
&& e1.errorMessage == e2.errorMessage && e1.line == e2.line
&& e1.column == e2.column && e1.codegenSuccessful == e2.codegenSuccessful;
};
// AotStats
// +-ModuleA
// | +-File1
// | | +-e1
// | | +-e2
// | +-File2
// | | +-e3
// +-ModuleB
// | +-File3
// | | +-e4
QQmlJS::AotStats original;
QQmlJS::AotStatsEntry e1 = createEntry(std::chrono::microseconds(500), "f1", "", 1, 1, true);
QQmlJS::AotStatsEntry e2 = createEntry(std::chrono::microseconds(200), "f2", "err1", 5, 4, false);
QQmlJS::AotStatsEntry e3 = createEntry(std::chrono::microseconds(750), "f3", "", 20, 4, true);
QQmlJS::AotStatsEntry e4 = createEntry(std::chrono::microseconds(300), "f4", "err2", 5, 8, false);
original.addEntry("ModuleA", "File1", e1);
original.addEntry("ModuleA", "File1", e2);
original.addEntry("ModuleA", "File2", e3);
original.addEntry("ModuleB", "File3", e4);
const auto parsed = QQmlJS::AotStats::fromJsonDocument(original.toJsonDocument());
QCOMPARE(parsed.entries().size(), original.entries().size());
const auto &parsedA = parsed.entries()["ModuleA"];
const auto &originalA = original.entries()["ModuleA"];
QCOMPARE(parsedA.size(), originalA.size());
QCOMPARE(parsedA["File1"].size(), originalA["File1"].size());
QVERIFY(equal(parsedA["File1"][0], originalA["File1"][0]));
QVERIFY(equal(parsedA["File1"][1], originalA["File1"][1]));
QCOMPARE(parsedA["File2"].size(), originalA["File2"].size());
QVERIFY(equal(parsedA["File2"][0], originalA["File2"][0]));
const auto &parsedB = parsed.entries()["ModuleB"];
const auto &originalB = original.entries()["ModuleB"];
QCOMPARE(parsedB.size(), originalB.size());
QCOMPARE(parsedB["File3"].size(), originalB["File3"].size());
QVERIFY(equal(parsedB["File3"][0], originalB["File3"][0]));
}
struct FunctionEntry
{
QString name;
QString errorMessage;
bool codegenSuccessful;
};
void tst_qmlcachegen::aotstatsGeneration_data()
{
QTest::addColumn<QString>("qmlFile");
QTest::addColumn<QList<FunctionEntry>>("entries");
QTest::addRow("clean") << "AotstatsClean.qml"
<< QList<FunctionEntry>{ { "j", "", true }, { "s", "", true } };
const QString fError = "function without return type annotation returns int. This may prevent "
"proper compilation to Cpp.";
const QString sError = "method g cannot be resolved.";
QTest::addRow("mixed") << "AotstatsMixed.qml"
<< QList<FunctionEntry>{ { "i", "", true },
{ "f", fError, false },
{ "s", sError, false } };
}
void tst_qmlcachegen::aotstatsGeneration()
{
#if defined(QTEST_CROSS_COMPILED)
QSKIP("Cannot call qmlcachegen on cross-compiled target.");
#endif
QFETCH(QString, qmlFile);
QFETCH(QList<FunctionEntry>, entries);
QTemporaryDir dir;
QProcess proc;
proc.setProgram(QLibraryInfo::path(QLibraryInfo::LibraryExecutablesPath) + "/qmlcachegen"_L1);
const QString cppOutput = dir.filePath(qmlFile + ".cpp");
const QString aotstatsOutput = cppOutput + ".aotstats";
proc.setArguments({ "--bare",
"--resource-path", "/cachegentest/data/aotstats/" + qmlFile,
"-i", testFile("aotstats/qmldir"),
"--resource", testFile("aotstats/cachegentest.qrc"),
"--dump-aot-stats",
"--module-id=Aotstats",
"-o", cppOutput,
testFile("aotstats/" + qmlFile) });
proc.start();
QVERIFY(proc.waitForFinished() && proc.exitStatus() == QProcess::NormalExit);
QVERIFY(QFileInfo::exists(aotstatsOutput));
QFile aotstatsFile(aotstatsOutput);
QVERIFY(aotstatsFile.open(QIODevice::Text | QIODevice::ReadOnly));
const auto document = QJsonDocument::fromJson(aotstatsFile.readAll());
const auto aotstats = QQmlJS::AotStats::fromJsonDocument(document);
QVERIFY(aotstats.entries().size() == 1); // One module
const auto &moduleEntries = aotstats.entries()["Aotstats"];
QVERIFY(moduleEntries.size() == 1); // Only one qml file was compiled
const auto &fileEntries = moduleEntries[moduleEntries.keys().first()];
for (const auto &entry : entries) {
const auto it = std::find_if(fileEntries.cbegin(), fileEntries.cend(),
[&](const auto &e) { return e.functionName == entry.name; });
QVERIFY(it != fileEntries.cend());
QVERIFY(it->codegenSuccessful == entry.codegenSuccessful);
QVERIFY(it->errorMessage == entry.errorMessage);
}
}
const QQmlScriptString &ScriptStringProps::undef() const
{
return m_undef;

18
tools/qmlcachegen/qmlcachegen.cpp
View File

@ -101,6 +101,11 @@ int main(int argc, char **argv)
QCommandLineOption validateBasicBlocksOption("validate-basic-blocks"_L1, QCoreApplication::translate("main", "Performs checks on the basic blocks of a function compiled ahead of time to validate its structure and coherence"));
parser.addOption(validateBasicBlocksOption);
QCommandLineOption dumpAotStatsOption("dump-aot-stats"_L1, QCoreApplication::translate("main", "Dumps statistics about ahead-of-time compilation of bindings and functions"));
parser.addOption(dumpAotStatsOption);
QCommandLineOption moduleIdOption("module-id"_L1, QCoreApplication::translate("main", "Identifies the module of the qml file being compiled for aot stats"), QCoreApplication::translate("main", "id"));
parser.addOption(moduleIdOption);
QCommandLineOption outputFileOption("o"_L1, QCoreApplication::translate("main", "Output file name"), QCoreApplication::translate("main", "file name"));
parser.addOption(outputFileOption);
@ -135,6 +140,11 @@ int main(int argc, char **argv)
if (target == GenerateLoader && parser.isSet(resourceNameOption))
target = GenerateLoaderStandAlone;
if (parser.isSet(dumpAotStatsOption) && !parser.isSet(moduleIdOption)) {
fprintf(stderr, "--dump-aot-stats set without setting --module-id");
return EXIT_FAILURE;
}
const QStringList sources = parser.positionalArguments();
if (sources.isEmpty()){
parser.showHelp();
@ -261,6 +271,11 @@ int main(int argc, char **argv)
&importer, u':' + inputResourcePath,
QQmlJSUtils::cleanPaths(parser.values(importsOption)), &logger);
if (parser.isSet(dumpAotStatsOption)) {
QQmlJS::QQmlJSAotCompilerStats::setRecordAotStats(true);
QQmlJS::QQmlJSAotCompilerStats::setModuleId(parser.value(moduleIdOption));
}
if (parser.isSet(validateBasicBlocksOption))
cppCodeGen.m_flags.setFlag(QQmlJSAotCompiler::ValidateBasicBlocks);
@ -279,6 +294,9 @@ int main(int argc, char **argv)
if (parser.isSet(warningsAreErrorsOption))
return EXIT_FAILURE;
}
if (parser.isSet(dumpAotStatsOption))
QQmlJS::QQmlJSAotCompilerStats::instance()->saveToDisk(outputFileName + u".aotstats"_s);
}
} else if (inputFile.endsWith(".js"_L1) || inputFile.endsWith(".mjs"_L1)) {
QQmlJSCompileError error;