Merge pull request #13 from RustPython/skip-notes

Show notes for skipped tests if they exist and add support for expectedFailure test results

Author: coolreader18

Gemfile.lock

@@ -81,4 +81,4 @@ DEPENDENCIES
 wdm (~> 0.1.1)
 
 BUNDLED WITH
-1.17.2
+2.1.4

_data/regrtests_results.json

@@ -1,11 +1,11 @@
-{% assign result_total = include.pass | plus: include.skip | plus: include.fail %}
+{% assign result_total = include.pass | plus: include.expectedfail | plus: include.skip %}
 {% assign pass_percent = include.pass | times: 100.0 | divided_by: result_total | round: 2 %}
+{% assign expectedfail_percent = include.expectedfail | times: 100.0 | divided_by: result_total | round: 2 %}
 {% assign skip_percent = include.skip | times: 100.0 | divided_by: result_total | round: 2 %}
-{% assign fail_percent = include.fail | times: 100.0 | divided_by: result_total | round: 2 %}
 
 {% assign pass_color = "#4c1" %}
-{% assign skip_color = "#e4df12" %}
-{% assign fail_color = "#fe7d37" %}
+{% assign expectedfail_color = "#e4df12" %}
+{% assign skip_color = "#fe7d37" %}
 {% assign bar_height = "20px" %}
 
 {%if include.svg %}
@@ -14,16 +14,16 @@
 <g shape-rendering="crispEdges">
 <rect width="49" height="20" fill="#555" />
 <rect x="49" width="65" height="20" fill="{{ pass_color }}" />
-<rect x="114" width="65" height="20" fill="{{ skip_color }}" />
-<rect x="179" width="65" height="20" fill="{{ fail_color }}" />
+<rect x="114" width="65" height="20" fill="{{ expectedfail_color }}" />
+<rect x="179" width="65" height="20" fill="{{ skip_color }}" />
 </g>
 <g fill="#fff" text-anchor="middle" font-family="Verdana,Geneva,DejaVu Sans,sans-serif"
 text-rendering="geometricPrecision" font-size="110">
 <text x="255" y="140" transform="scale(.1)"
 textLength="390">Results</text>
 <text x="805" y="140" transform="scale(.1)" textLength="550">Pass {{ pass_percent | round: 1 }}%</text>
-<text x="1455" y="140" transform="scale(.1)" textLength="550">Skip {{ skip_percent | round: 1 }}%</text>
-<text x="2105" y="140" transform="scale(.1)" textLength="550">Fail {{ fail_percent | round: 1 }}%</text>
+<text x="1455" y="140" transform="scale(.1)" textLength="550">Expected Failure {{ expectedfail_percent | round: 1 }}%</text>
+<text x="2105" y="140" transform="scale(.1)" textLength="550">Skip {{ skip_percent | round: 1 }}%</text>
 </g>
 </svg>
 
@@ -50,20 +50,20 @@
 <span class="font-secondary">Passed: </span>
 {{ include.pass }} ({{ pass_percent }}%)
 </div>
+<div class ="cell_summary">
+<span class="font-secondary">Expected Failure: </span>
+{{ include.expectedfail }} ({{ expectedfail_percent }}%)
+</div>
 <div class="cell_summary">
 <span class="font-secondary">Skipped: </span>
 {{ include.skip }} ({{ skip_percent }}%)
 </div>
-<div class ="cell_summary">
-<span class="font-secondary">Failed: </span>
-{{ include.fail }} ({{ fail_percent }}%)
-</div>
 </div>
 <div class="row"
 style="background: linear-gradient(to right,
 {{ pass_color }} {{ pass_percent }}%,
-{{ skip_color }} {{ pass_percent }}% {{ skip_percent | plus: pass_percent }}%,
-{{ fail_color }} {{ skip_percent | plus: pass_percent }}% {{ fail_percent | plus: pass_percent | plus: skip_percent }}%
+{{ expectedfail_color }} {{ pass_percent }}% {{ expectedfail_percent | plus: pass_percent }}%,
+{{ skip_color }} {{ expectedfail_percent | plus: pass_percent }}% {{ skip_percent | plus: pass_percent | plus: expectedfail_percent }}%
 );
 height: {{ bar_height }};">
 

_includes/regrtest_summary.html

@@ -20,11 +20,13 @@
 <div data-filter="all" class="filter active pr-2"> All </div>
 <div data-filter="passed" class="filter pr-2"> Passed </div>
 <div data-filter="skipped" class="filter pr-2"> Skipped </div>
-<div data-filter="" class="filter"> Empty </div>
+<div data-filter="expected_failure" class="filter pr-2"> Expected Failures </div>
+<div data-filter="" class="filter ml-auto"> Summary only </div>
 </div>
 </div>
 
 <!-- content -->
+{% capture tables %}
 <div class="w-80 m-auto">
 <div class="m-auto">
 
@@ -36,8 +38,8 @@ <h2 class="font-secondary">Summary:</span>
 {% assign regrtests_results = site.data.regrtests_results %}
 {% include regrtest_summary.html
 pass=regrtests_results.num_passed
+expectedfail=regrtests_results.num_expected_failures
 skip=regrtests_results.num_skipped
-fail=regrtests_results.num_failed
 extime=regrtests_results.execution_time
 %}
 </div>
@@ -76,7 +78,7 @@ <h2 class="font-secondary">Summary:</span>
 </div>
 {% if result.num_passed and result.num_skipped and result.num_failed %}
 <div class="w-md-50 float-md-right">
-{% include regrtest_summary.html pass=result.num_passed skip=result.num_skipped fail=result.num_failed extime=result.execution_time %}
+{% include regrtest_summary.html pass=result.num_passed expectedfail=result.num_expected_failures skip=result.num_skipped extime=result.execution_time %}
 </div>
 {% endif %}
 </div>
@@ -85,7 +87,7 @@ <h2 class="font-secondary">Summary:</span>
 {% for cases in result.cases %}
 <div class="results-case">
 <div class="tbl results-case-list">
-<!-- for each case -->
+{% comment %} for each case {% endcomment %}
 {% for case in cases %}
 <div data-status="{{case.result}}" class="row results-case-list-details">
 
@@ -97,9 +99,25 @@ <h2 class="font-secondary">Summary:</span>
 {% endif %}
 
 {% if case.result %}
-<div class="cell {{case.result }} results-case-list-item text-mobile-right">
-<span class="text-small text-muted d-mobile-none">RESULT:</span>
-{{case.result }}
+<div class="cell {{ case.result }} results-case-list-item text-mobile-right">
+<span class="text-small text-muted d-mobile-none">RESULT: </span>
+<span class="case-result">{{ case.result | replace: "_", " " }}</span>
+{%- if case.result == "skipped" -%}
+{%- assign note = case.note | default: result.note-%}
+{%- assign lownote = note | downcase -%}
+{%- if lownote contains "todo: rustpython" -%}
+{%- assign note = lownote | remove: "todo: rustpython," | remove: "todo: rustpython:" | remove: "todo: rustpython" | strip -%}
+{%- if note and note != "" %}
+<span class="text-small text-muted d-mobile-none">TO RESOLVE: </span>
+<span class="d-mobile-none case-note">{{ note }}</span>
+{%- endif -%}
+{%- elsif note == "implementation detail specific to cpython" -%}
+<span class="text-small text-muted d-mobile-none">(CPython only)</span>
+{%- else -%}
+<span class="text-small text-muted d-mobile-none">not skipped by us: </span>
+<span class="text-small d-mobile-none case-note">{{ note }}</span>
+{%- endif -%}
+{%- endif -%}
 </div>
 {% endif %}
 
@@ -112,7 +130,7 @@ <h2 class="font-secondary">Summary:</span>
 
 </div>
 {% endfor %}
-<!-- end for each case -->
+{% comment %} end for each case {% endcomment %}
 </div>
 </div>
 {% endfor %}
@@ -124,5 +142,10 @@ <h2 class="font-secondary">Summary:</span>
 {% endfor %}
 </div>
 </div>
+{% endcapture %}
+
+{%- assign tables = tables | split: "
+"-%}
+{%- for line in tables -%}{{ line | strip }}{%- endfor -%}
 
 <script src="/assets/js/regrtests_results.js"></script>

_layouts/regrtests_results.html

@@ -13,6 +13,8 @@ Let's find an incompatibility issue and fix it.
 
 1. See `Lib/test` directory of the project. There are many `test_` prefixed files like `test_unicode.py`.
 2. Try to open one of them. It might look just fine at a glance - but search for `TODO: RUSTPYTHON` in the files. There are tons of skipped, marked as an expected failure or commented out tests.
+1. Alternatively, try looking at the [regression tests results]({% link pages/regression-tests-results.markdown %}) to find skipped or expected failure tests; some of them have
+notes for a way to resolve the issue.
 3. Choose one or two interesting bugs. Remove the test blocker - skip, expectedFailure or comments.
 4. Try to fix them.
 

_posts/2020-04-05-how-to-contribute-by-cpython-unittest.markdown

@@ -80,6 +80,9 @@
 text-align: right;
 }
 
+.ml-auto {
+margin-left: auto !important;
+}
 }
 
 /* fixes for ipad */

assets/media.css

@@ -315,13 +315,20 @@ ul.list-inline {
 padding: 5px;
 }
 
-.passed::after {
-content: " 🚀 👌";
+.case-result::after {
 padding: 5px;
 }
 
-.skipped::after {
-content: " 🤷‍♀️";
+.passed > .case-result::after {
+content: "🚀 👌";
+}
+
+.skipped > .case-result::after {
+content: "🤷‍♀️";
+}
+
+.expected_failure > .case-result::after {
+content: "😔";
 }
 
 /* filters */

assets/style.css

@@ -36,6 +36,10 @@ Each of these implementations offer some benefits: Jython, for example, compiles
 
 IronPython is well-integrated with .NET, which means IronPython can use the .NET framework and Python libraries or vice versa.
 
-We want to unlock the same possibilities that Jython and IronPython enable, but for the Rust programming language. Then we want to do more. Using RustPython, we want to compile Python to WebAssembly and allow users to run their Python code natively in the browser.
+We want to unlock the same possibilities that Jython and IronPython enable, but for the Rust programming language. In addition, thanks to Rusts' minimal runtime, we're able to compile RustPython to WebAssembly and allow users to run their Python code easily in the browser.
 
 Check the "learn more" section for an explainer of all those jargon-y words, or read the blog for more in-depth technical discussion.
+
+### Other Resources
+
+- [CPython test compatibility]({% link pages/regression-tests-results.markdown %})