Skip to content

Commit

Permalink
Generate a markdown summary of test failures along with existing
Browse files Browse the repository at this point in the history 
HTML report
  • Loading branch information
@nyalldawson
nyalldawson committed Oct 11, 2023
1 parent d3b96bc commit 415c39e
Show file tree
Hide file tree
Showing 7 changed files with 143 additions and 9 deletions.
15 changes: 14 additions & 1 deletion python/core/auto_generated/qgsmultirenderchecker.sip.in
View file
Original file line number Diff line number Diff line change
Expand Up @@ -124,9 +124,22 @@ Test using renderer to generate the image to be compared.

QString report() const;
%Docstring
Returns a report for this test.
Returns a HTML report for this test.

The report will be empty if the test was successfully run.

.. seealso:: :py:func:`markdownReport`
%End

QString markdownReport() const;
%Docstring
Returns a markdown report for this test.

The report will be empty if the test was successfully run.

.. seealso:: :py:func:`report`

.. versionadded:: 3.34
%End

QString controlImagePath() const;
Expand Down
14 changes: 14 additions & 0 deletions python/core/auto_generated/qgsrenderchecker.sip.in
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,20 @@ Returns the HTML report describing the results of the test run.

If ``ignoreSuccess`` is ``True`` then the report will always be empty if
the test was successful.

.. seealso:: :py:func:`markdownReport`
%End

QString markdownReport( bool ignoreSuccess = true ) const;
%Docstring
Returns the markdown report describing the results of the test run.

If ``ignoreSuccess`` is ``True`` then the report will always be empty if
the test was successful.

.. seealso:: :py:func:`report`

.. versionadded:: 3.34
%End

float matchPercent() const;
Expand Down
10 changes: 10 additions & 0 deletions src/core/qgsmultirenderchecker.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ bool QgsMultiRenderChecker::runTest( const QString &testName, unsigned int misma
mResult = false;

mReport += "<h2>" + testName + "</h2>\n";
mMarkdownReport += QStringLiteral( "### %1\n\n" ).arg( testName );

const QString baseDir = controlImagePath();
if ( !QFile::exists( baseDir ) )
Expand Down Expand Up @@ -97,6 +98,10 @@ bool QgsMultiRenderChecker::runTest( const QString &testName, unsigned int misma
dartMeasurements << checker.dartMeasurements();

mReport += checker.report( false );
if ( subDirs.count() > 1 )
mMarkdownReport += QStringLiteral( "* " ) + checker.markdownReport( false );
else
mMarkdownReport += checker.markdownReport( false );

if ( !mResult && diffImageFile.isEmpty() )
{
Expand Down Expand Up @@ -173,6 +178,11 @@ QString QgsMultiRenderChecker::report() const
return !mResult ? mReport : QString();
}

QString QgsMultiRenderChecker::markdownReport() const
{
return !mResult ? mMarkdownReport : QString();
}

QString QgsMultiRenderChecker::controlImagePath() const
{
QString myDataDir( TEST_DATA_DIR ); //defined in CmakeLists.txt
Expand Down
15 changes: 14 additions & 1 deletion src/core/qgsmultirenderchecker.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -128,12 +128,24 @@ class CORE_EXPORT QgsMultiRenderChecker
bool runTest( const QString &testName, unsigned int mismatchCount = 0 );

/**
* Returns a report for this test.
* Returns a HTML report for this test.
*
* The report will be empty if the test was successfully run.
*
* \see markdownReport()
*/
QString report() const;

/**
* Returns a markdown report for this test.
*
* The report will be empty if the test was successfully run.
*
* \see report()
* \since QGIS 3.34
*/
QString markdownReport() const;

/**
* Returns the path to the control images.
*/
Expand All @@ -148,6 +160,7 @@ class CORE_EXPORT QgsMultiRenderChecker
private:
bool mResult = false;
QString mReport;
QString mMarkdownReport;
QString mRenderedImage;
QString mControlName;
QString mControlPathPrefix;
Expand Down
26 changes: 26 additions & 0 deletions src/core/qgsrenderchecker.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,11 @@ QString QgsRenderChecker::report( bool ignoreSuccess ) const
return ( ( ignoreSuccess && mResult ) || ( mExpectFail && !mResult ) ) ? QString() : mReport;
}

QString QgsRenderChecker::markdownReport( bool ignoreSuccess ) const
{
return ( ( ignoreSuccess && mResult ) || ( mExpectFail && !mResult ) ) ? QString() : mMarkdownReport;
}

void QgsRenderChecker::setControlName( const QString &name )
{
mControlName = name;
Expand Down Expand Up @@ -252,6 +257,7 @@ bool QgsRenderChecker::runTest( const QString &testName,
"<tr><td>Test Result:</td><td>Expected Result:</td></tr>\n"
"<tr><td>Nothing rendered</td>\n<td>Failed because Expected "
"Image File not set.</td></tr></table>\n";
mMarkdownReport = QStringLiteral( "Failed because expected image file not set\n" );
performPostTestActions( flags );
return mResult;
}
Expand All @@ -266,6 +272,7 @@ bool QgsRenderChecker::runTest( const QString &testName,
"<tr><td>Test Result:</td><td>Expected Result:</td></tr>\n"
"<tr><td>Nothing rendered</td>\n<td>Failed because Expected "
"Image File could not be loaded.</td></tr></table>\n";
mMarkdownReport = QStringLiteral( "Failed because expected image file (%1) could not be loaded\n" ).arg( mExpectedImageFile );
performPostTestActions( flags );
return mResult;
}
Expand Down Expand Up @@ -304,6 +311,8 @@ bool QgsRenderChecker::runTest( const QString &testName,
"<tr><td>Test Result:</td><td>Expected Result:</td></tr>\n"
"<tr><td>Nothing rendered</td>\n<td>Failed because Rendered "
"Image File could not be saved.</td></tr></table>\n";
mMarkdownReport = QStringLiteral( "Failed because rendered image file could not be saved to %1\n" ).arg( mRenderedImageFile );

performPostTestActions( flags );
return mResult;
}
Expand Down Expand Up @@ -340,6 +349,8 @@ bool QgsRenderChecker::compareImages( const QString &testName,
"<tr><td>Test Result:</td><td>Expected Result:</td></tr>\n"
"<tr><td>Nothing rendered</td>\n<td>Failed because Expected "
"Image File not set.</td></tr></table>\n";
mMarkdownReport = QStringLiteral( "Failed because expected image file was not set\n" );

performPostTestActions( flags );
return mResult;
}
Expand All @@ -365,6 +376,7 @@ bool QgsRenderChecker::compareImages( const QString &testName, const QString &re
"<tr><td>Test Result:</td><td>Expected Result:</td></tr>\n"
"<tr><td>Nothing rendered</td>\n<td>Failed because Rendered "
"Image File not set.</td></tr></table>\n";
mMarkdownReport = QStringLiteral( "Failed because rendered image file was not set\n" );
performPostTestActions( flags );
return mResult;
}
Expand All @@ -380,6 +392,7 @@ bool QgsRenderChecker::compareImages( const QString &testName, const QString &re
"<tr><td>Test Result:</td><td>Expected Result:</td></tr>\n"
"<tr><td>Nothing rendered</td>\n<td>Failed because control "
"image file could not be loaded.</td></tr></table>\n";
mMarkdownReport = QStringLiteral( "Failed because expected image file (%1) could not be loaded\n" ).arg( referenceImageFile );
performPostTestActions( flags );
return mResult;
}
Expand All @@ -400,6 +413,7 @@ bool QgsRenderChecker::compareImages( const QString &testName, const QString &re
"<tr><td>Test Result:</td><td>%1:</td></tr>\n"
"<tr><td>Nothing rendered</td>\n<td>Failed because Rendered "
"Image File could not be loaded.</td></tr></table>\n" ).arg( upperFirst( expectedImageString ) );
mMarkdownReport = QStringLiteral( "Failed because rendered image (%1) could not be loaded\n" ).arg( mRenderedImageFile );
performPostTestActions( flags );
return mResult;
}
Expand Down Expand Up @@ -508,6 +522,11 @@ bool QgsRenderChecker::compareImages( const QString &testName, const QString &re
mReport += QLatin1String( "<tr><td colspan=3>" );
mReport += QStringLiteral( "<font color=red>%1 and %2 for " ).arg( upperFirst( expectedImageString ), renderedImageString ) + testName + " are different dimensions - FAILING!</font>";
mReport += QLatin1String( "</td></tr>" );
mMarkdownReport += QStringLiteral( "Failed because rendered image and expected image are different dimensions (%1x%2 v2 %3x%4)\n" )
.arg( myResultImage.width() )
.arg( myResultImage.height() )
.arg( expectedImage.width() )
.arg( expectedImage.height() );

const QString diffSizeImagesString = QString(
"<tr>"
Expand Down Expand Up @@ -548,6 +567,8 @@ bool QgsRenderChecker::compareImages( const QString &testName, const QString &re
mReport += "<font color=red>Expected image and rendered image for " + testName + " have different formats (8bit format is expected) - FAILING!</font>";
mReport += QLatin1String( "</td></tr>" );
mReport += myImagesString;

mMarkdownReport += QStringLiteral( "Failed because rendered image and expected image have different formats (8bit format is expected)\n" );
performPostTestActions( flags );
return mResult;
}
Expand Down Expand Up @@ -655,6 +676,9 @@ bool QgsRenderChecker::compareImages( const QString &testName, const QString &re
mReport += QLatin1String( "<font color=red>Test failed because render step took too long</font>" );
mReport += QLatin1String( "</td></tr>" );
mReport += myImagesString;

mMarkdownReport += QStringLiteral( "Test failed because render step took too long\n" );

performPostTestActions( flags );
return mResult;
}
Expand All @@ -680,6 +704,8 @@ bool QgsRenderChecker::compareImages( const QString &testName, const QString &re
mReport += QLatin1String( "</td></tr>" );
mReport += myImagesString;

mMarkdownReport += QStringLiteral( "Rendered image did not match %1 (found %2 pixels different)\n" ).arg( referenceImageFile ).arg( mMismatchCount );

performPostTestActions( flags );
return mResult;
}
16 changes: 16 additions & 0 deletions src/core/qgsrenderchecker.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,9 +86,22 @@ class CORE_EXPORT QgsRenderChecker
*
* If \a ignoreSuccess is TRUE then the report will always be empty if
* the test was successful.
*
* \see markdownReport()
*/
QString report( bool ignoreSuccess = true ) const;

/**
* Returns the markdown report describing the results of the test run.
*
* If \a ignoreSuccess is TRUE then the report will always be empty if
* the test was successful.
*
* \see report()
* \since QGIS 3.34
*/
QString markdownReport( bool ignoreSuccess = true ) const;

/**
* Returns the percent of pixels which matched the control image.
*/
Expand Down Expand Up @@ -277,7 +290,10 @@ class CORE_EXPORT QgsRenderChecker
QVector<QgsDartMeasurement> dartMeasurements() const { return mDashMessages; }

protected:
//! HTML format report
QString mReport;
//! Markdown report
QString mMarkdownReport;
unsigned int mMatchTarget = 0;
int mElapsedTime = 0;
QString mRenderedImageFile;
Expand Down
56 changes: 49 additions & 7 deletions src/test/qgstest.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,8 @@ class TEST_EXPORT QgsTest : public QObject
{
if ( !mReport.isEmpty() )
writeLocalHtmlReport( mReport );
if ( !mMarkdownReport.isEmpty() )
writeMarkdownReport( mMarkdownReport );
}

/**
Expand Down Expand Up @@ -264,7 +266,7 @@ class TEST_EXPORT QgsTest : public QObject
const bool result = checker.runTest( name, allowedMismatch );
if ( !result )
{
appendToReport( name, checker.report() );
appendToReport( name, checker.report(), checker.markdownReport() );

}
return result;
Expand All @@ -290,7 +292,7 @@ class TEST_EXPORT QgsTest : public QObject
const bool result = checker.runTest( name, allowedMismatch );
if ( !result )
{
appendToReport( name, checker.report() );
appendToReport( name, checker.report(), checker.markdownReport() );
}
return result;
}
Expand All @@ -308,29 +310,42 @@ class TEST_EXPORT QgsTest : public QObject
const bool result = checker.testLayout( report, page, allowedMismatch );
if ( !result )
{
appendToReport( name, report );
appendToReport( name, report, checker.markdownReport() );
}
return result;
}

/**
* Appends some \a content to the test report.
* Appends some \a html and \a markdown to the test report.
*
* This should be used only for appending useful information when a test fails.
*/
void appendToReport( const QString &testName, const QString &content )
void appendToReport( const QString &testName, const QString &html, const QString &markdown = QString() )
{
QString testIdentifier;
if ( QTest::currentDataTag() )
testIdentifier = QStringLiteral( "%1 (%2: %3)" ).arg( testName, QTest::currentTestFunction(), QTest::currentDataTag() );
else
testIdentifier = QStringLiteral( "%1 (%2)" ).arg( testName, QTest::currentTestFunction() );
mReport += QStringLiteral( "<h2>%1</h2>\n" ).arg( testIdentifier );
mReport += content;

if ( !html.isEmpty() )
{
mReport += QStringLiteral( "<h2>%1</h2>\n" ).arg( testIdentifier );
mReport += html;
}

const QString markdownContent = markdown.isEmpty() ? html : markdown;
if ( !markdownContent.isEmpty() )
{
mMarkdownReport += QStringLiteral( "## %1\n\n" ).arg( testIdentifier );
mMarkdownReport += markdownContent + QStringLiteral( "\n\n" );
}
}

private:

QString mMarkdownReport;

/**
* Writes out a HTML report to a temporary file for visual comparison
* of test results on a local build.
Expand Down Expand Up @@ -363,6 +378,33 @@ class TEST_EXPORT QgsTest : public QObject
}
}

/**
* Writes out a markdown report to a temporary file for use on CI runs.
*/
void writeMarkdownReport( const QString &report )
{
const QDir reportDir = QgsRenderChecker::testReportDir();
if ( !reportDir.exists() )
QDir().mkpath( reportDir.path() );

const QString reportFile = reportDir.filePath( "summary.md" );
QFile file( reportFile );

QFile::OpenMode mode = QIODevice::WriteOnly;
if ( qgetenv( "QGIS_CONTINUOUS_INTEGRATION_RUN" ) == QStringLiteral( "true" )
|| qgetenv( "QGIS_APPEND_TO_TEST_REPORT" ) == QStringLiteral( "true" ) )
mode |= QIODevice::Append;
else
mode |= QIODevice::Truncate;

if ( file.open( mode ) )
{
QTextStream stream( &file );
stream << report;
file.close();
}
}

};

/**
Expand Down

0 comments on commit 415c39e

Please sign in to comment.