Fix *.js files disappearing from RESOURCES when not using QtQml

[ChangeLog][QtQml][Important Behavior Changes] Using the Qt Quick
Compiler would exclude the original .qml files from the resource system.
This made it impossible to change the Qt library binary later as the
program binary was tied the to the exact Qt version. In addition
sometimes unrelated files (QTBUG-73669) were removed. For the latter
scenario, retain and skip options were added for the Qt Quick Compiler.
In Qt 5.15 the Qt Quick Compiler does not remove the input files
anymore. All files are retained and the compiler merely adds the more
efficient binary representation to the application.

Task-number: QTBUG-73669
Change-Id: I5a523bfc69d4f48a1451bd880616c82fd73b8d15
Reviewed-by: Ulf Hermann <ulf.hermann@qt.io>
Reviewed-by: Fabian Kosmale <fabian.kosmale@qt.io>
This commit is contained in:
Simon Hausmann 2019-08-27 16:12:38 +02:00
parent c716753454
commit 41864db3b6
7 changed files with 18 additions and 483 deletions

View File

@ -360,7 +360,7 @@ static QQmlPrivate::CachedQmlUnit *temporaryModifiedCachedUnit = nullptr;
void tst_qmlcachegen::versionChecksForAheadOfTimeUnits()
{
QVERIFY(QFile::exists(":/data/versionchecks.qml"));
QCOMPARE(QFileInfo(":/data/versionchecks.qml").size(), 0);
QVERIFY(QFileInfo(":/data/versionchecks.qml").size() > 0);
Q_ASSERT(!temporaryModifiedCachedUnit);
QQmlMetaType::CachedUnitLookupError error = QQmlMetaType::CachedUnitLookupError::NoError;
@ -387,12 +387,8 @@ void tst_qmlcachegen::versionChecksForAheadOfTimeUnits()
{
QQmlEngine engine;
QQmlComponent component(&engine, QUrl("qrc:/data/versionchecks.qml"));
QCOMPARE(component.status(), QQmlComponent::Error);
QCOMPARE(component.errorString(),
QString("qrc:/data/versionchecks.qml:-1 File was compiled ahead of time with an "
"incompatible version of Qt and the original file cannot be found. Please "
"recompile\n"));
CleanlyLoadingComponent component(&engine, QUrl("qrc:/data/versionchecks.qml"));
QCOMPARE(component.status(), QQmlComponent::Ready);
}
Q_ASSERT(temporaryModifiedCachedUnit);
@ -414,7 +410,7 @@ void tst_qmlcachegen::workerScripts()
{
QVERIFY(QFile::exists(":/workerscripts/data/worker.js"));
QVERIFY(QFile::exists(":/workerscripts/data/worker.qml"));
QCOMPARE(QFileInfo(":/workerscripts/data/worker.js").size(), 0);
QVERIFY(QFileInfo(":/workerscripts/data/worker.js").size() > 0);
QQmlEngine engine;
CleanlyLoadingComponent component(&engine, QUrl("qrc:///workerscripts/data/worker.qml"));
@ -503,7 +499,7 @@ void tst_qmlcachegen::trickyPaths()
{
QFETCH(QString, filePath);
QVERIFY2(QFile::exists(filePath), qPrintable(filePath));
QCOMPARE(QFileInfo(filePath).size(), 0);
QVERIFY(QFileInfo(filePath).size() > 0);
QQmlEngine engine;
QQmlComponent component(&engine, QUrl("qrc" + filePath));
QScopedPointer<QObject> obj(component.create());
@ -584,7 +580,7 @@ void tst_qmlcachegen::moduleScriptImport()
QTRY_VERIFY(obj->property("ok").toBool());
QVERIFY(QFile::exists(":/data/script.mjs"));
QCOMPARE(QFileInfo(":/data/script.mjs").size(), 0);
QVERIFY(QFileInfo(":/data/script.mjs").size() > 0);
{
auto componentPrivate = QQmlComponentPrivate::get(&component);
@ -617,7 +613,7 @@ void tst_qmlcachegen::enums()
void tst_qmlcachegen::sourceFileIndices()
{
QVERIFY(QFile::exists(":/data/versionchecks.qml"));
QCOMPARE(QFileInfo(":/data/versionchecks.qml").size(), 0);
QVERIFY(QFileInfo(":/data/versionchecks.qml").size() > 0);
QQmlMetaType::CachedUnitLookupError error = QQmlMetaType::CachedUnitLookupError::NoError;
const QV4::CompiledData::Unit *unitFromResources = QQmlMetaType::findCachedCompilationUnit(

View File

@ -50,13 +50,9 @@ but not all the files it references.
get_filename_component(input_resource ${_resource} ABSOLUTE)
execute_process(COMMAND ${compiler_path} -filter-resource-file ${input_resource} -o ${new_resource_file} OUTPUT_VARIABLE remaining_files)
if(remaining_files)
list(APPEND filtered_rcc_files ${new_resource_file})
list(APPEND loader_flags \"--resource-file-mapping=${_resource}=${new_resource_file}\")
else()
list(APPEND loader_flags \"--resource-file-mapping=${_resource}\")
endif()
configure_file(${input_resource} ${new_resource_file} COPYONLY)
list(APPEND filtered_rcc_files ${new_resource_file})
list(APPEND loader_flags \"--resource-file-mapping=${_resource}=${new_resource_file}\")
set(rcc_file_with_compilation_units)

View File

@ -100,228 +100,6 @@ QString symbolNamespaceForPath(const QString &relativePath)
return mangledIdentifier(symbol);
}
struct VirtualDirectoryEntry
{
QString name;
QVector<VirtualDirectoryEntry*> dirEntries;
int firstChildIndex = -1; // node index inside generated data
bool isDirectory = true;
VirtualDirectoryEntry()
{}
~VirtualDirectoryEntry()
{
qDeleteAll(dirEntries);
}
VirtualDirectoryEntry *append(const QString &name)
{
for (QVector<VirtualDirectoryEntry*>::Iterator it = dirEntries.begin(), end = dirEntries.end();
it != end; ++it) {
if ((*it)->name == name)
return *it;
}
VirtualDirectoryEntry *subEntry = new VirtualDirectoryEntry;
subEntry->name = name;
dirEntries.append(subEntry);
return subEntry;
}
void appendEmptyFile(const QString &name)
{
VirtualDirectoryEntry *subEntry = new VirtualDirectoryEntry;
subEntry->name = name;
subEntry->isDirectory = false;
dirEntries.append(subEntry);
}
bool isEmpty() const { return dirEntries.isEmpty(); }
};
struct DataStream
{
DataStream(QVector<unsigned char > *data = nullptr)
: data(data)
{}
qint64 currentOffset() const { return data->size(); }
DataStream &operator<<(quint16 value)
{
unsigned char d[2];
qToBigEndian(value, d);
data->append(d[0]);
data->append(d[1]);
return *this;
}
DataStream &operator<<(quint32 value)
{
unsigned char d[4];
qToBigEndian(value, d);
data->append(d[0]);
data->append(d[1]);
data->append(d[2]);
data->append(d[3]);
return *this;
}
private:
QVector<unsigned char> *data;
};
static bool resource_sort_order(const VirtualDirectoryEntry *lhs, const VirtualDirectoryEntry *rhs)
{
return qt_hash(lhs->name) < qt_hash(rhs->name);
}
struct ResourceTree
{
ResourceTree()
{}
void serialize(VirtualDirectoryEntry &root, QVector<unsigned char> *treeData, QVector<unsigned char> *stringData)
{
treeStream = DataStream(treeData);
stringStream = DataStream(stringData);
QStack<VirtualDirectoryEntry *> directories;
{
directories.push(&root);
while (!directories.isEmpty()) {
VirtualDirectoryEntry *entry = directories.pop();
registerString(entry->name);
if (entry->isDirectory)
directories << entry->dirEntries;
}
}
{
quint32 currentDirectoryIndex = 1;
directories.push(&root);
while (!directories.isEmpty()) {
VirtualDirectoryEntry *entry = directories.pop();
entry->firstChildIndex = currentDirectoryIndex;
currentDirectoryIndex += entry->dirEntries.count();
std::sort(entry->dirEntries.begin(), entry->dirEntries.end(), resource_sort_order);
for (QVector<VirtualDirectoryEntry*>::ConstIterator child = entry->dirEntries.constBegin(), end = entry->dirEntries.constEnd();
child != end; ++child) {
if ((*child)->isDirectory)
directories << *child;
}
}
}
{
writeTreeEntry(&root);
directories.push(&root);
while (!directories.isEmpty()) {
VirtualDirectoryEntry *entry = directories.pop();
for (QVector<VirtualDirectoryEntry*>::ConstIterator child = entry->dirEntries.constBegin(), end = entry->dirEntries.constEnd();
child != end; ++child) {
writeTreeEntry(*child);
if ((*child)->isDirectory)
directories << (*child);
}
}
}
}
private:
DataStream treeStream;
DataStream stringStream;
QHash<QString, qint64> stringOffsets;
void registerString(const QString &name)
{
if (stringOffsets.contains(name))
return;
const qint64 offset = stringStream.currentOffset();
stringOffsets.insert(name, offset);
stringStream << quint16(name.length())
<< quint32(qt_hash(name));
for (int i = 0; i < name.length(); ++i)
stringStream << quint16(name.at(i).unicode());
}
void writeTreeEntry(VirtualDirectoryEntry *entry)
{
treeStream << quint32(stringOffsets.value(entry->name))
<< quint16(entry->isDirectory ? 0x2 : 0x0); // Flags: File or Directory
if (entry->isDirectory) {
treeStream << quint32(entry->dirEntries.count())
<< quint32(entry->firstChildIndex);
} else {
treeStream << quint16(QLocale::AnyCountry) << quint16(QLocale::C)
<< quint32(0x0);
}
}
};
static QByteArray generateResourceDirectoryTree(QTextStream &code, const QStringList &qrcFiles,
const QStringList &sortedRetainedFiles)
{
QByteArray call;
if (qrcFiles.isEmpty())
return call;
VirtualDirectoryEntry resourceDirs;
resourceDirs.name = QStringLiteral("/");
for (const QString &entry : qrcFiles) {
const QStringList segments = entry.split(QLatin1Char('/'), QString::SkipEmptyParts);
VirtualDirectoryEntry *dirEntry = &resourceDirs;
for (int i = 0; i < segments.count() - 1; ++i)
dirEntry = dirEntry->append(segments.at(i));
if (!std::binary_search(sortedRetainedFiles.begin(), sortedRetainedFiles.end(), entry))
dirEntry->appendEmptyFile(segments.last());
}
if (resourceDirs.isEmpty())
return call;
QVector<unsigned char> names;
QVector<unsigned char> tree;
ResourceTree().serialize(resourceDirs, &tree, &names);
code << "static const unsigned char qt_resource_tree[] = {\n";
for (int i = 0; i < tree.count(); ++i) {
code << uint(tree.at(i));
if (i < tree.count() - 1)
code << ',';
if (i % 16 == 0)
code << '\n';
}
code << "};\n";
code << "static const unsigned char qt_resource_names[] = {\n";
for (int i = 0; i < names.count(); ++i) {
code << uint(names.at(i));
if (i < names.count() - 1)
code << ',';
if (i % 16 == 0)
code << '\n';
}
code << "};\n";
code << "static const unsigned char qt_resource_empty_payout[] = { 0, 0, 0, 0, 0 };\n";
code << "QT_BEGIN_NAMESPACE\n";
code << "extern Q_CORE_EXPORT bool qRegisterResourceData(int, const unsigned char *, const unsigned char *, const unsigned char *);\n";
code << "QT_END_NAMESPACE\n";
call = "QT_PREPEND_NAMESPACE(qRegisterResourceData)(/*version*/0x01, qt_resource_tree, qt_resource_names, qt_resource_empty_payout);\n";
return call;
}
static QString qtResourceNameForFile(const QString &fileName)
{
QFileInfo fi(fileName);
@ -332,9 +110,8 @@ static QString qtResourceNameForFile(const QString &fileName)
return name;
}
bool generateLoader(const QStringList &compiledFiles, const QStringList &sortedRetainedFiles,
const QString &outputFileName, const QStringList &resourceFileMappings,
QString *errorString)
bool generateLoader(const QStringList &compiledFiles, const QString &outputFileName,
const QStringList &resourceFileMappings, QString *errorString)
{
QByteArray generatedLoaderCode;
@ -345,9 +122,6 @@ bool generateLoader(const QStringList &compiledFiles, const QStringList &sortedR
stream << "#include <QtCore/qurl.h>\n";
stream << "\n";
QByteArray resourceRegisterCall = generateResourceDirectoryTree(stream, compiledFiles,
sortedRetainedFiles);
stream << "namespace QmlCacheGeneratedCode {\n";
for (int i = 0; i < compiledFiles.count(); ++i) {
const QString compiledFile = compiledFiles.at(i);
@ -385,9 +159,6 @@ bool generateLoader(const QStringList &compiledFiles, const QStringList &sortedR
stream << " registration.lookupCachedQmlUnit = &lookupCachedUnit;\n";
stream << " QQmlPrivate::qmlregister(QQmlPrivate::QmlUnitCacheHookRegistration, &registration);\n";
if (!resourceRegisterCall.isEmpty())
stream << resourceRegisterCall;
stream << "}\n\n";
stream << "Registry::~Registry() {\n";
stream << " QQmlPrivate::qmlunregister(QQmlPrivate::QmlUnitCacheHookRegistration, quintptr(&lookupCachedUnit));\n";

View File

@ -47,10 +47,8 @@
using namespace QQmlJS;
int filterResourceFile(const QString &input, const QString &output);
bool generateLoader(const QStringList &compiledFiles, const QStringList &retainedFiles,
const QString &output, const QStringList &resourceFileMappings,
QString *errorString);
bool generateLoader(const QStringList &compiledFiles, const QString &output,
const QStringList &resourceFileMappings, QString *errorString);
QString symbolNamespaceForPath(const QString &relativePath);
QSet<QString> illegalNames;
@ -419,14 +417,10 @@ int main(int argc, char **argv)
parser.addHelpOption();
parser.addVersionOption();
QCommandLineOption filterResourceFileOption(QStringLiteral("filter-resource-file"), QCoreApplication::translate("main", "Filter out QML/JS files from a resource file that can be cached ahead of time instead"));
parser.addOption(filterResourceFileOption);
QCommandLineOption resourceFileMappingOption(QStringLiteral("resource-file-mapping"), QCoreApplication::translate("main", "Path from original resource file to new one"), QCoreApplication::translate("main", "old-name:new-name"));
parser.addOption(resourceFileMappingOption);
QCommandLineOption resourceOption(QStringLiteral("resource"), QCoreApplication::translate("main", "Qt resource file that might later contain one of the compiled files"), QCoreApplication::translate("main", "resource-file-name"));
parser.addOption(resourceOption);
QCommandLineOption retainOption(QStringLiteral("retain"), QCoreApplication::translate("main", "Qt resource file the contents of which should not be replaced by empty stubs"), QCoreApplication::translate("main", "resource-file-name"));
parser.addOption(retainOption);
QCommandLineOption resourcePathOption(QStringLiteral("resource-path"), QCoreApplication::translate("main", "Qt resource file path corresponding to the file being compiled"), QCoreApplication::translate("main", "resource-path"));
parser.addOption(resourcePathOption);
@ -468,18 +462,11 @@ int main(int argc, char **argv)
if (outputFileName.isEmpty())
outputFileName = inputFile + QLatin1Char('c');
if (parser.isSet(filterResourceFileOption)) {
return filterResourceFile(inputFile, outputFileName);
}
if (target == GenerateLoader) {
ResourceFileMapper mapper(sources);
ResourceFileMapper retain(parser.values(retainOption));
Error error;
QStringList retainedFiles = retain.qmlCompilerFiles();
std::sort(retainedFiles.begin(), retainedFiles.end());
if (!generateLoader(mapper.qmlCompilerFiles(), retainedFiles, outputFileName,
if (!generateLoader(mapper.qmlCompilerFiles(), outputFileName,
parser.values(resourceFileMappingOption), &error.message)) {
error.augment(QLatin1String("Error generating loader stub: ")).print();
return EXIT_FAILURE;

View File

@ -4,7 +4,6 @@ QT = qmldevtools-private
DEFINES += QT_NO_CAST_TO_ASCII QT_NO_CAST_FROM_ASCII
SOURCES = qmlcachegen.cpp \
resourcefilter.cpp \
generateloader.cpp \
resourcefilemapper.cpp
TARGET = qmlcachegen

View File

@ -16,20 +16,6 @@ defineReplace(qmlCacheResourceFileOutputName) {
return($${name})
}
defineTest(qtQuickRetainSources) {
for(retainedRes, QTQUICK_COMPILER_RETAINED_RESOURCES) {
equals(1, $$retainedRes): return(true)
}
return(false)
}
defineTest(qtQuickSkippedResourceFile) {
for(skippedRes, QTQUICK_COMPILER_SKIPPED_RESOURCES) {
equals(1, $$skippedRes): return(true)
}
return(false)
}
# Flatten RESOURCES that may contain individual files or objects
load(resources)
@ -37,29 +23,14 @@ NEWRESOURCES =
QMLCACHE_RESOURCE_FILES =
for(res, RESOURCES) {
qtQuickSkippedResourceFile($$res) {
NEWRESOURCES += $$res
next()
}
absRes = $$absolute_path($$res, $$_PRO_FILE_PWD_)
rccContents = $$system($$QMAKE_RCC_DEP -list $$system_quote($$absRes),lines)
contains(rccContents,.*\\.js$)|contains(rccContents,.*\\.qml$)|contains(rccContents,.*\\.mjs$) {
new_resource = $$qmlCacheResourceFileOutputName($$res)
mkpath($$dirname(new_resource))
qtQuickRetainSources($$res) {
NEWRESOURCES += $$res
QMLCACHE_LOADER_FLAGS += --retain=$$shell_quote($$absRes)
} else {
remaining_files = $$system($$QML_CACHEGEN_FILTER -filter-resource-file \
-o $$system_quote($$new_resource) $$system_quote($$absRes),lines)
!isEmpty(remaining_files) {
NEWRESOURCES += $$new_resource
QMLCACHE_LOADER_FLAGS += --resource-file-mapping=$$shell_quote($$absRes=$$new_resource)
} else {
QMLCACHE_LOADER_FLAGS += --resource-file-mapping=$$shell_quote($$absRes)
}
}
system($$QMAKE_QMAKE -install qinstall $$system_quote($$absRes) $$system_quote($$new_resource))
NEWRESOURCES += $$new_resource
QMLCACHE_LOADER_FLAGS += --resource-file-mapping=$$shell_quote($$absRes=$$new_resource)
QMLCACHE_RESOURCE_FILES += $$absRes

View File

@ -1,185 +0,0 @@
/****************************************************************************
**
** Copyright (C) 2016 The Qt Company Ltd.
** Contact: https://www.qt.io/licensing/
**
** This file is part of the QtQml module of the Qt Toolkit.
**
** $QT_BEGIN_LICENSE:GPL-EXCEPT$
** Commercial License Usage
** Licensees holding valid commercial Qt licenses may use this file in
** accordance with the commercial license agreement provided with the
** Software or, alternatively, in accordance with the terms contained in
** a written agreement between you and The Qt Company. For licensing terms
** and conditions see https://www.qt.io/terms-conditions. For further
** information use the contact form at https://www.qt.io/contact-us.
**
** GNU General Public License Usage
** Alternatively, this file may be used under the terms of the GNU
** General Public License version 3 as published by the Free Software
** Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
** included in the packaging of this file. Please review the following
** information to ensure the GNU General Public License requirements will
** be met: https://www.gnu.org/licenses/gpl-3.0.html.
**
** $QT_END_LICENSE$
**
****************************************************************************/
#include <QString>
#include <QXmlStreamReader>
#include <QFile>
#include <QDir>
int filterResourceFile(const QString &input, const QString &output)
{
enum State {
InitialState,
InRCC,
InResource,
InFile
};
State state = InitialState;
QString prefix;
QString currentFileName;
QXmlStreamAttributes fileAttributes;
QFile file(input);
if (!file.open(QIODevice::ReadOnly)) {
fprintf(stderr, "Cannot open %s for reading.\n", qPrintable(input));
return EXIT_FAILURE;
}
QDir inputDirectory = QFileInfo(file).absoluteDir();
QDir outputDirectory = QFileInfo(output).absoluteDir();
QString outputString;
QXmlStreamWriter writer(&outputString);
writer.setAutoFormatting(true);
QStringList remainingFiles;
QXmlStreamReader reader(&file);
while (!reader.atEnd()) {
switch (reader.readNext()) {
case QXmlStreamReader::StartDocument: {
QStringRef version = reader.documentVersion();
if (!version.isEmpty())
writer.writeStartDocument(version.toString());
else
writer.writeStartDocument();
break;
}
case QXmlStreamReader::EndDocument:
writer.writeEndDocument();
break;
case QXmlStreamReader::StartElement:
if (reader.name() == QStringLiteral("RCC")) {
if (state != InitialState) {
fprintf(stderr, "Unexpected RCC tag in line %d\n", int(reader.lineNumber()));
return EXIT_FAILURE;
}
state = InRCC;
} else if (reader.name() == QStringLiteral("qresource")) {
if (state != InRCC) {
fprintf(stderr, "Unexpected qresource tag in line %d\n", int(reader.lineNumber()));
return EXIT_FAILURE;
}
state = InResource;
QXmlStreamAttributes attributes = reader.attributes();
if (attributes.hasAttribute(QStringLiteral("prefix")))
prefix = attributes.value(QStringLiteral("prefix")).toString();
if (!prefix.startsWith(QLatin1Char('/')))
prefix.prepend(QLatin1Char('/'));
if (!prefix.endsWith(QLatin1Char('/')))
prefix.append(QLatin1Char('/'));
} else if (reader.name() == QStringLiteral("file")) {
if (state != InResource) {
fprintf(stderr, "Unexpected file tag in line %d\n", int(reader.lineNumber()));
return EXIT_FAILURE;
}
state = InFile;
fileAttributes = reader.attributes();
continue;
}
writer.writeStartElement(reader.name().toString());
writer.writeAttributes(reader.attributes());
continue;
case QXmlStreamReader::EndElement:
if (reader.name() == QStringLiteral("file")) {
if (state != InFile) {
fprintf(stderr, "Unexpected end of file tag in line %d\n", int(reader.lineNumber()));
return EXIT_FAILURE;
}
state = InResource;
continue;
} else if (reader.name() == QStringLiteral("qresource")) {
if (state != InResource) {
fprintf(stderr, "Unexpected end of qresource tag in line %d\n", int(reader.lineNumber()));
return EXIT_FAILURE;
}
state = InRCC;
} else if (reader.name() == QStringLiteral("RCC")) {
if (state != InRCC) {
fprintf(stderr, "Unexpected end of RCC tag in line %d\n", int(reader.lineNumber()));
return EXIT_FAILURE;
}
state = InitialState;
}
writer.writeEndElement();
continue;
case QXmlStreamReader::Characters:
if (reader.isWhitespace())
break;
if (state != InFile)
return EXIT_FAILURE;
currentFileName = reader.text().toString();
if (currentFileName.isEmpty())
continue;
if (!currentFileName.endsWith(QStringLiteral(".qml"))
&& !currentFileName.endsWith(QStringLiteral(".js"))
&& !currentFileName.endsWith(QStringLiteral(".mjs"))) {
writer.writeStartElement(QStringLiteral("file"));
if (!fileAttributes.hasAttribute(QStringLiteral("alias")))
fileAttributes.append(QStringLiteral("alias"), currentFileName);
currentFileName = inputDirectory.absoluteFilePath(currentFileName);
currentFileName = outputDirectory.relativeFilePath(currentFileName);
remainingFiles << currentFileName;
writer.writeAttributes(fileAttributes);
writer.writeCharacters(currentFileName);
writer.writeEndElement();
}
continue;
default: break;
}
}
if (!remainingFiles.isEmpty()) {
QFile outputFile(output);
if (!outputFile.open(QIODevice::WriteOnly | QIODevice::Truncate)) {
fprintf(stderr, "Cannot open %s for writing.\n", qPrintable(output));
return EXIT_FAILURE;
}
const QByteArray outputStringUtf8 = outputString.toUtf8();
if (outputFile.write(outputStringUtf8) != outputStringUtf8.size())
return EXIT_FAILURE;
outputFile.close();
if (outputFile.error() != QFileDevice::NoError)
return EXIT_FAILURE;
// The build system expects this output if we wrote a qrc file and no output
// if no files remain.
fprintf(stdout, "New resource file written with %d files.\n", remainingFiles.count());
}
return EXIT_SUCCESS;
}