Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Output information about skipped tests in xml output #2767

Merged
merged 5 commits into from
May 28, 2020
Merged

Output information about skipped tests in xml output #2767

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
59e5b40
Output skipped information in the xml file.
mvoorsluys Mar 26, 2020
d28d05c
Only write ">\n" once when there is failure and skipped tests.
mvoorsluys Mar 26, 2020
23dadb8
Fix multiple \n characters in xml file when using GTEST_SKIP.
mvoorsluys Mar 26, 2020
c447b21
Fixed xml unit-tests and added extra tests
mvoorsluys Mar 26, 2020
7c8ab52
Fix test with stack.
mvoorsluys Mar 26, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 27 additions & 6 deletions googletest/src/gtest.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2191,7 +2191,9 @@ std::string AppendUserMessage(const std::string& gtest_msg,
if (user_msg_string.empty()) {
return gtest_msg;
}

if (gtest_msg.empty()) {
return user_msg_string;
}
return gtest_msg + "\n" + user_msg_string;
}

Expand Down Expand Up @@ -2270,7 +2272,8 @@ static const char* const kReservedTestSuitesAttributes[] = {
// The list of reserved attributes used in the <testsuite> element of XML
// output.
static const char* const kReservedTestSuiteAttributes[] = {
"disabled", "errors", "failures", "name", "tests", "time", "timestamp"};
"disabled", "errors", "failures", "name",
"tests", "time", "timestamp", "skipped"};

// The list of reserved attributes used in the <testcase> element of XML output.
static const char* const kReservedTestCaseAttributes[] = {
Expand Down Expand Up @@ -3112,7 +3115,7 @@ static std::string FormatTestSuiteCount(int test_suite_count) {
static const char * TestPartResultTypeToString(TestPartResult::Type type) {
switch (type) {
case TestPartResult::kSkip:
return "Skipped";
return "Skipped\n";
case TestPartResult::kSuccess:
return "Success";

Expand Down Expand Up @@ -4080,10 +4083,11 @@ void XmlUnitTestResultPrinter::OutputXmlTestInfo(::std::ostream* stream,
OutputXmlAttribute(stream, kTestsuite, "classname", test_suite_name);

int failures = 0;
int skips = 0;
for (int i = 0; i < result.total_part_count(); ++i) {
const TestPartResult& part = result.GetTestPartResult(i);
if (part.failed()) {
if (++failures == 1) {
if (++failures == 1 && skips == 0) {
*stream << ">\n";
}
const std::string location =
Expand All @@ -4096,13 +4100,26 @@ void XmlUnitTestResultPrinter::OutputXmlTestInfo(::std::ostream* stream,
const std::string detail = location + "\n" + part.message();
OutputXmlCDataSection(stream, RemoveInvalidXmlCharacters(detail).c_str());
*stream << "</failure>\n";
} else if (part.skipped()) {
if (++skips == 1 && failures == 0) {
*stream << ">\n";
}
const std::string location =
internal::FormatCompilerIndependentFileLocation(part.file_name(),
part.line_number());
const std::string summary = location + "\n" + part.summary();
*stream << " <skipped message=\""
<< EscapeXmlAttribute(summary.c_str()) << "\">";
const std::string detail = location + "\n" + part.message();
OutputXmlCDataSection(stream, RemoveInvalidXmlCharacters(detail).c_str());
*stream << "</skipped>\n";
}
}

if (failures == 0 && result.test_property_count() == 0) {
if (failures == 0 && skips == 0 && result.test_property_count() == 0) {
*stream << " />\n";
} else {
if (failures == 0) {
if (failures == 0 && skips == 0) {
*stream << ">\n";
}
OutputXmlTestProperties(stream, result);
Expand All @@ -4124,7 +4141,11 @@ void XmlUnitTestResultPrinter::PrintXmlTestSuite(std::ostream* stream,
OutputXmlAttribute(
stream, kTestsuite, "disabled",
StreamableToString(test_suite.reportable_disabled_test_count()));
OutputXmlAttribute(stream, kTestsuite, "skipped",
StreamableToString(test_suite.skipped_test_count()));

OutputXmlAttribute(stream, kTestsuite, "errors", "0");

OutputXmlAttribute(stream, kTestsuite, "time",
FormatTimeInMillisAsSeconds(test_suite.elapsed_time()));
OutputXmlAttribute(
Expand Down
28 changes: 24 additions & 4 deletions googletest/test/googletest-json-output-unittest.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,9 +58,9 @@

EXPECTED_NON_EMPTY = {
u'tests':
24,
26,
u'failures':
4,
5,
u'disabled':
2,
u'errors':
Expand Down Expand Up @@ -158,9 +158,9 @@
u'name':
u'SkippedTest',
u'tests':
1,
3,
u'failures':
0,
1,
u'disabled':
0,
u'errors':
Expand All @@ -176,6 +176,26 @@
u'time': u'*',
u'timestamp': u'*',
u'classname': u'SkippedTest'
}, {
u'name': u'SkippedWithMessage',
u'status': u'RUN',
u'result': u'SKIPPED',
u'time': u'*',
u'timestamp': u'*',
u'classname': u'SkippedTest'
}, {
u'name': u'SkippedAfterFailure',
u'status': u'RUN',
u'result': u'COMPLETED',
u'time': u'*',
u'timestamp': u'*',
u'classname': u'SkippedTest',
u'failures': [{
u'failure': u'gtest_xml_output_unittest_.cc:*\n'
u'Expected equality of these values:\n'
u' 1\n 2' + STACK_TRACE_TEMPLATE,
u'type': u''
}]
}]
}, {
u'name':
Expand Down
4 changes: 2 additions & 2 deletions googletest/test/gtest_xml_outfiles_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@

EXPECTED_XML_1 = """<?xml version="1.0" encoding="UTF-8"?>
<testsuites tests="1" failures="0" disabled="0" errors="0" time="*" timestamp="*" name="AllTests">
<testsuite name="PropertyOne" tests="1" failures="0" disabled="0" errors="0" time="*" timestamp="*">
<testsuite name="PropertyOne" tests="1" failures="0" skipped="0" disabled="0" errors="0" time="*" timestamp="*">
<testcase name="TestSomeProperties" status="run" result="completed" time="*" timestamp="*" classname="PropertyOne">
<properties>
<property name="SetUpProp" value="1"/>
Expand All @@ -56,7 +56,7 @@

EXPECTED_XML_2 = """<?xml version="1.0" encoding="UTF-8"?>
<testsuites tests="1" failures="0" disabled="0" errors="0" time="*" timestamp="*" name="AllTests">
<testsuite name="PropertyTwo" tests="1" failures="0" disabled="0" errors="0" time="*" timestamp="*">
<testsuite name="PropertyTwo" tests="1" failures="0" skipped="0" disabled="0" errors="0" time="*" timestamp="*">
<testcase name="TestSomeProperties" status="run" result="completed" time="*" timestamp="*" classname="PropertyTwo">
<properties>
<property name="SetUpProp" value="2"/>
Expand Down
62 changes: 39 additions & 23 deletions googletest/test/gtest_xml_output_unittest.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,19 +65,19 @@
sys.argv.remove(NO_STACKTRACE_SUPPORT_FLAG)

EXPECTED_NON_EMPTY_XML = """<?xml version="1.0" encoding="UTF-8"?>
<testsuites tests="24" failures="4" disabled="2" errors="0" time="*" timestamp="*" name="AllTests" ad_hoc_property="42">
<testsuite name="SuccessfulTest" tests="1" failures="0" disabled="0" errors="0" time="*" timestamp="*">
<testsuites tests="26" failures="5" disabled="2" errors="0" time="*" timestamp="*" name="AllTests" ad_hoc_property="42">
<testsuite name="SuccessfulTest" tests="1" failures="0" disabled="0" skipped="0" errors="0" time="*" timestamp="*">
<testcase name="Succeeds" status="run" result="completed" time="*" timestamp="*" classname="SuccessfulTest"/>
</testsuite>
<testsuite name="FailedTest" tests="1" failures="1" disabled="0" errors="0" time="*" timestamp="*">
<testsuite name="FailedTest" tests="1" failures="1" disabled="0" skipped="0" errors="0" time="*" timestamp="*">
<testcase name="Fails" status="run" result="completed" time="*" timestamp="*" classname="FailedTest">
<failure message="gtest_xml_output_unittest_.cc:*&#x0A;Expected equality of these values:&#x0A; 1&#x0A; 2" type=""><![CDATA[gtest_xml_output_unittest_.cc:*
Expected equality of these values:
1
2%(stack)s]]></failure>
</testcase>
</testsuite>
<testsuite name="MixedResultTest" tests="3" failures="1" disabled="1" errors="0" time="*" timestamp="*">
<testsuite name="MixedResultTest" tests="3" failures="1" disabled="1" skipped="0" errors="0" time="*" timestamp="*">
<testcase name="Succeeds" status="run" result="completed" time="*" timestamp="*" classname="MixedResultTest"/>
<testcase name="Fails" status="run" result="completed" time="*" timestamp="*" classname="MixedResultTest">
<failure message="gtest_xml_output_unittest_.cc:*&#x0A;Expected equality of these values:&#x0A; 1&#x0A; 2" type=""><![CDATA[gtest_xml_output_unittest_.cc:*
Expand All @@ -91,27 +91,43 @@
</testcase>
<testcase name="DISABLED_test" status="notrun" result="suppressed" time="*" timestamp="*" classname="MixedResultTest"/>
</testsuite>
<testsuite name="XmlQuotingTest" tests="1" failures="1" disabled="0" errors="0" time="*" timestamp="*">
<testsuite name="XmlQuotingTest" tests="1" failures="1" disabled="0" skipped="0" errors="0" time="*" timestamp="*">
<testcase name="OutputsCData" status="run" result="completed" time="*" timestamp="*" classname="XmlQuotingTest">
<failure message="gtest_xml_output_unittest_.cc:*&#x0A;Failed&#x0A;XML output: &lt;?xml encoding=&quot;utf-8&quot;&gt;&lt;top&gt;&lt;![CDATA[cdata text]]&gt;&lt;/top&gt;" type=""><![CDATA[gtest_xml_output_unittest_.cc:*
Failed
XML output: <?xml encoding="utf-8"><top><![CDATA[cdata text]]>]]&gt;<![CDATA[</top>%(stack)s]]></failure>
</testcase>
</testsuite>
<testsuite name="InvalidCharactersTest" tests="1" failures="1" disabled="0" errors="0" time="*" timestamp="*">
<testsuite name="InvalidCharactersTest" tests="1" failures="1" disabled="0" skipped="0" errors="0" time="*" timestamp="*">
<testcase name="InvalidCharactersInMessage" status="run" result="completed" time="*" timestamp="*" classname="InvalidCharactersTest">
<failure message="gtest_xml_output_unittest_.cc:*&#x0A;Failed&#x0A;Invalid characters in brackets []" type=""><![CDATA[gtest_xml_output_unittest_.cc:*
Failed
Invalid characters in brackets []%(stack)s]]></failure>
</testcase>
</testsuite>
<testsuite name="DisabledTest" tests="1" failures="0" disabled="1" errors="0" time="*" timestamp="*">
<testsuite name="DisabledTest" tests="1" failures="0" disabled="1" skipped="0" errors="0" time="*" timestamp="*">
<testcase name="DISABLED_test_not_run" status="notrun" result="suppressed" time="*" timestamp="*" classname="DisabledTest"/>
</testsuite>
<testsuite name="SkippedTest" tests="1" failures="0" disabled="0" errors="0" time="*" timestamp="*">
<testcase name="Skipped" status="run" result="skipped" time="*" timestamp="*" classname="SkippedTest"/>
<testsuite name="SkippedTest" tests="3" failures="1" disabled="0" skipped="2" errors="0" time="*" timestamp="*">
<testcase name="Skipped" status="run" result="skipped" time="*" timestamp="*" classname="SkippedTest">
<skipped message="gtest_xml_output_unittest_.cc:*&#x0A;"><![CDATA[gtest_xml_output_unittest_.cc:*
%(stack)s]]></skipped>
</testcase>
<testcase name="SkippedWithMessage" status="run" result="skipped" time="*" timestamp="*" classname="SkippedTest">
<skipped message="gtest_xml_output_unittest_.cc:*&#x0A;It is good practice to tell why you skip a test."><![CDATA[gtest_xml_output_unittest_.cc:*
It is good practice to tell why you skip a test.%(stack)s]]></skipped>
</testcase>
<testcase name="SkippedAfterFailure" status="run" result="completed" time="*" timestamp="*" classname="SkippedTest">
<failure message="gtest_xml_output_unittest_.cc:*&#x0A;Expected equality of these values:&#x0A; 1&#x0A; 2" type=""><![CDATA[gtest_xml_output_unittest_.cc:*
Expected equality of these values:
1
2%(stack)s]]></failure>
<skipped message="gtest_xml_output_unittest_.cc:*&#x0A;It is good practice to tell why you skip a test."><![CDATA[gtest_xml_output_unittest_.cc:*
It is good practice to tell why you skip a test.%(stack)s]]></skipped>
</testcase>

</testsuite>
<testsuite name="PropertyRecordingTest" tests="4" failures="0" disabled="0" errors="0" time="*" timestamp="*" SetUpTestSuite="yes" TearDownTestSuite="aye">
<testsuite name="PropertyRecordingTest" tests="4" failures="0" disabled="0" skipped="0" errors="0" time="*" timestamp="*" SetUpTestSuite="yes" TearDownTestSuite="aye">
<testcase name="OneProperty" status="run" result="completed" time="*" timestamp="*" classname="PropertyRecordingTest">
<properties>
<property name="key_1" value="1"/>
Expand All @@ -135,7 +151,7 @@
</properties>
</testcase>
</testsuite>
<testsuite name="NoFixtureTest" tests="3" failures="0" disabled="0" errors="0" time="*" timestamp="*">
<testsuite name="NoFixtureTest" tests="3" failures="0" disabled="0" skipped="0" errors="0" time="*" timestamp="*">
<testcase name="RecordProperty" status="run" result="completed" time="*" timestamp="*" classname="NoFixtureTest">
<properties>
<property name="key" value="1"/>
Expand All @@ -152,22 +168,22 @@
</properties>
</testcase>
</testsuite>
<testsuite name="Single/ValueParamTest" tests="4" failures="0" disabled="0" errors="0" time="*" timestamp="*">
<testsuite name="Single/ValueParamTest" tests="4" failures="0" disabled="0" skipped="0" errors="0" time="*" timestamp="*">
<testcase name="HasValueParamAttribute/0" value_param="33" status="run" result="completed" time="*" timestamp="*" classname="Single/ValueParamTest" />
<testcase name="HasValueParamAttribute/1" value_param="42" status="run" result="completed" time="*" timestamp="*" classname="Single/ValueParamTest" />
<testcase name="AnotherTestThatHasValueParamAttribute/0" value_param="33" status="run" result="completed" time="*" timestamp="*" classname="Single/ValueParamTest" />
<testcase name="AnotherTestThatHasValueParamAttribute/1" value_param="42" status="run" result="completed" time="*" timestamp="*" classname="Single/ValueParamTest" />
</testsuite>
<testsuite name="TypedTest/0" tests="1" failures="0" disabled="0" errors="0" time="*" timestamp="*">
<testsuite name="TypedTest/0" tests="1" failures="0" disabled="0" skipped="0" errors="0" time="*" timestamp="*">
<testcase name="HasTypeParamAttribute" type_param="*" status="run" result="completed" time="*" timestamp="*" classname="TypedTest/0" />
</testsuite>
<testsuite name="TypedTest/1" tests="1" failures="0" disabled="0" errors="0" time="*" timestamp="*">
<testsuite name="TypedTest/1" tests="1" failures="0" disabled="0" skipped="0" errors="0" time="*" timestamp="*">
<testcase name="HasTypeParamAttribute" type_param="*" status="run" result="completed" time="*" timestamp="*" classname="TypedTest/1" />
</testsuite>
<testsuite name="Single/TypeParameterizedTestSuite/0" tests="1" failures="0" disabled="0" errors="0" time="*" timestamp="*">
<testsuite name="Single/TypeParameterizedTestSuite/0" tests="1" failures="0" disabled="0" skipped="0" errors="0" time="*" timestamp="*">
<testcase name="HasTypeParamAttribute" type_param="*" status="run" result="completed" time="*" timestamp="*" classname="Single/TypeParameterizedTestSuite/0" />
</testsuite>
<testsuite name="Single/TypeParameterizedTestSuite/1" tests="1" failures="0" disabled="0" errors="0" time="*" timestamp="*">
<testsuite name="Single/TypeParameterizedTestSuite/1" tests="1" failures="0" disabled="0" skipped="0" errors="0" time="*" timestamp="*">
<testcase name="HasTypeParamAttribute" type_param="*" status="run" result="completed" time="*" timestamp="*" classname="Single/TypeParameterizedTestSuite/1" />
</testsuite>
</testsuites>""" % {
Expand All @@ -177,26 +193,26 @@
EXPECTED_FILTERED_TEST_XML = """<?xml version="1.0" encoding="UTF-8"?>
<testsuites tests="1" failures="0" disabled="0" errors="0" time="*"
timestamp="*" name="AllTests" ad_hoc_property="42">
<testsuite name="SuccessfulTest" tests="1" failures="0" disabled="0"
<testsuite name="SuccessfulTest" tests="1" failures="0" disabled="0" skipped="0"
errors="0" time="*" timestamp="*">
<testcase name="Succeeds" status="run" result="completed" time="*" timestamp="*" classname="SuccessfulTest"/>
</testsuite>
</testsuites>"""

EXPECTED_SHARDED_TEST_XML = """<?xml version="1.0" encoding="UTF-8"?>
<testsuites tests="3" failures="0" disabled="0" errors="0" time="*" timestamp="*" name="AllTests" ad_hoc_property="42">
<testsuite name="SuccessfulTest" tests="1" failures="0" disabled="0" errors="0" time="*" timestamp="*">
<testsuite name="SuccessfulTest" tests="1" failures="0" disabled="0" skipped="0" errors="0" time="*" timestamp="*">
<testcase name="Succeeds" status="run" result="completed" time="*" timestamp="*" classname="SuccessfulTest"/>
</testsuite>
<testsuite name="PropertyRecordingTest" tests="1" failures="0" disabled="0" errors="0" time="*" timestamp="*" SetUpTestSuite="yes" TearDownTestSuite="aye">
<testcase name="TwoValuesForOneKeyUsesLastValue" status="run" result="completed" time="*" timestamp="*" classname="PropertyRecordingTest">
<testsuite name="PropertyRecordingTest" tests="1" failures="0" disabled="0" skipped="0" errors="0" time="*" timestamp="*" SetUpTestSuite="yes" TearDownTestSuite="aye">
<testcase name="IntValuedProperty" status="run" result="completed" time="*" timestamp="*" classname="PropertyRecordingTest">
<properties>
<property name="key_1" value="2"/>
<property name="key_int" value="1"/>
</properties>
</testcase>
</testsuite>
<testsuite name="Single/ValueParamTest" tests="1" failures="0" disabled="0" errors="0" time="*" timestamp="*">
<testcase name="AnotherTestThatHasValueParamAttribute/0" value_param="33" status="run" result="completed" time="*" timestamp="*" classname="Single/ValueParamTest" />
<testsuite name="Single/ValueParamTest" tests="1" failures="0" disabled="0" skipped="0" errors="0" time="*" timestamp="*">
<testcase name="HasValueParamAttribute/0" value_param="33" status="run" result="completed" time="*" timestamp="*" classname="Single/ValueParamTest" />
</testsuite>
</testsuites>"""

Expand Down
9 changes: 9 additions & 0 deletions googletest/test/gtest_xml_output_unittest_.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,15 @@ TEST_F(SkippedTest, Skipped) {
GTEST_SKIP();
}

TEST_F(SkippedTest, SkippedWithMessage) {
GTEST_SKIP() << "It is good practice to tell why you skip a test.";
}

TEST_F(SkippedTest, SkippedAfterFailure) {
EXPECT_EQ(1, 2);
GTEST_SKIP() << "It is good practice to tell why you skip a test.";
}

TEST(MixedResultTest, Succeeds) {
EXPECT_EQ(1, 1);
ASSERT_EQ(1, 1);
Expand Down
3 changes: 2 additions & 1 deletion googletest/test/gtest_xml_test_utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,7 @@ def AssertEquivalentNodes(self, expected_node, actual_node):
'testsuite': 'name',
'testcase': 'name',
'failure': 'message',
'skipped': 'message',
'property': 'name',
}

Expand Down Expand Up @@ -179,7 +180,7 @@ def NormalizeXml(self, element):
type_param = element.getAttributeNode('type_param')
if type_param and type_param.value:
type_param.value = '*'
elif element.tagName == 'failure':
elif element.tagName == 'failure' or element.tagName == 'skipped':
source_line_pat = r'^.*[/\\](.*:)\d+\n'
# Replaces the source line information with a normalized form.
message = element.getAttributeNode('message')
Expand Down