ci: use previous test timings to split the tests

.github/actions/ci-save-split-tests/action.yml

@@ -0,0 +1,17 @@
+name: 'Save split-tests'
+description: 'Save Junit test results and timing data, to better split future tests'
+
+inputs:
+  results_path:
+    description: 'Glob pattern to the JUnit files to save'
+    required: true
+
+# This should be run once the results from all runs are collected.
+runs:
+  using: composite
+  steps:
+    - name: Save test reports
+      uses: actions/cache@v2
+      with:
+        path: ${{ inputs.results_path }}
+        key: tests-reports-${{ github.ref }}-${{ github.sha }}-${{ github.run_id }}

.github/actions/ci-setup-rails/action.yml

@@ -29,9 +29,3 @@ runs:
         DATABASE_URL: "postgres://tps_test@localhost:5432/tps_test"
       run: bin/rails db:create db:schema:load db:migrate
       shell: bash
-
-    - name: Setup split_tests binary
-      run: |
-        curl --no-progress-meter -L https://github.com/leonid-shevtsov/split_tests/releases/download/v0.3.0/split_tests.linux.gz | gunzip -c > split_tests
-        chmod +x split_tests
-      shell: bash

.github/actions/ci-setup-split-tests/action.yml

@@ -0,0 +1,46 @@
+name: 'Setup split-tests'
+description: 'Setup the environment for splitting tests'
+
+inputs:
+  results_path:
+    description: 'Glob pattern to the JUnit files to save'
+    required: true
+
+runs:
+  using: composite
+  steps:
+    - name: Setup split_tests binary
+      run: |
+        curl --no-progress-meter -L https://github.com/leonid-shevtsov/split_tests/releases/download/v0.3.0/split_tests.linux.gz | gunzip -c > split_tests
+        chmod +x split_tests
+      shell: bash
+
+    - name: Generate an unique random value
+      run: echo dummy_random_value=$RANDOM >> $GITHUB_ENV
+      shell: bash
+
+    # Restore previous runs timing from the cache.
+    #
+    # NB: at the end of the job, the `actions/cache@v2` action will attempt
+    # to save the results back to the same key. However at this stage only
+    # tests results for a single instance will be available.
+    #
+    # To avoid the cache being overwritten with this single result, we define
+    # a random cache key, which the action will use to save the single-instance
+    # report to a dummy location.
+    #
+    # The actual retrieval uses the `restore-keys` instead.
+    - name: Restore previous runs timings
+      uses: actions/cache@v2
+      with:
+        path: ${{ inputs.results_path }}
+        key: single-instance-report-${{ github.sha }}-${{ env.dummy_random_value }}
+        restore-keys: |
+          tests-reports-${{ github.ref }}-${{ github.sha }}-${{ github.run_id }}
+          tests-reports-${{ github.ref }}-${{ github.sha }}-
+          tests-reports-${{ github.ref }}-
+          tests-reports-
+
+    - name: Display previous runs timings used for splitting tests
+      run: ls ${{ inputs.results_path }} || true
+      shell: bash

.github/workflows/ci.yml

@@ -43,7 +43,7 @@ jobs:
 
     strategy:
       matrix:
-        split_index: [0, 1, 2, 3, 4, 5]
+        instances: [0, 1, 2, 3, 4, 5]
 
     steps:
       - uses: actions/checkout@v2
@@ -54,11 +54,22 @@ jobs:
       - name: Pre-compile assets
         uses: ./.github/actions/ci-setup-assets
 
+      - name: Setup split tests
+        uses: ./.github/actions/ci-setup-split-tests
+        with:
+          results_path: tmp/*.junit.xml
+
       - name: Run tests
         run: |
-          SPEC_FILES=$(./split_tests -line-count -glob='spec/**/*_spec.rb' -exclude-glob='spec/system/**' -split-index=${{ matrix.split_index }} -split-total=6)
+          SPEC_FILES=$(./split_tests -glob='spec/**/*_spec.rb' -exclude-glob='spec/system/**' -split-index=${{ strategy.job-index }} -split-total=${{ strategy.job-total }} -junit -junit-path=tmp/*.junit.xml)
           echo "Running tests for bin/rspec $SPEC_FILES"
-          bin/rspec $SPEC_FILES
+          bin/rspec $SPEC_FILES --format progress --format RspecJunitFormatter --out tmp/rspec_${{ github.job }}_${{ strategy.job-index }}.junit.xml
+
+      - name: Upload test results for this instance
+        uses: actions/upload-artifact@v2
+        with:
+          name: test-reports
+          path: tmp/rspec_${{ github.job }}_${{ strategy.job-index }}.junit.xml
 
   system_tests:
     name: System tests
@@ -74,7 +85,7 @@ jobs:
 
     strategy:
       matrix:
-        split_index: [0, 1]
+        instances: [0, 1]
 
     steps:
       - uses: actions/checkout@v2
@@ -85,8 +96,38 @@ jobs:
       - name: Pre-compile assets
         uses: ./.github/actions/ci-setup-assets
 
+      - name: Setup split tests
+        uses: ./.github/actions/ci-setup-split-tests
+        with:
+          results_path: tmp/*.junit.xml
+
       - name: Run tests
         run: |
-          SPEC_FILES=$(./split_tests -line-count -glob='spec/system/**/*_spec.rb' -split-index=${{ matrix.split_index }} -split-total=2)
+          SPEC_FILES=$(./split_tests -glob='spec/system/**/*_spec.rb' -split-index=${{ strategy.job-index }} -split-total=${{ strategy.job-total }} -junit -junit-path=tmp/*.junit.xml)
           echo "Running tests for bin/rspec $SPEC_FILES"
-          bin/rspec $SPEC_FILES
+          bin/rspec $SPEC_FILES --format progress --format RspecJunitFormatter --out tmp/rspec_${{ github.job }}_${{ strategy.job-index }}.junit.xml
+
+      - name: Upload test results for this instance
+        uses: actions/upload-artifact@v2
+        with:
+          name: test-reports
+          path: tmp/rspec_${{ github.job }}_${{ strategy.job-index }}.junit.xml
+
+  save_test_reports:
+    name: Save test reports
+    needs: [unit_tests, system_tests]
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Collect test results from all instances
+        uses: actions/download-artifact@v2
+        with:
+          name: test-reports
+          path: tmp
+
+      - name: Save test results and timing data, to better split future tests
+        uses: ./.github/actions/ci-save-split-tests
+        with:
+          results_path: tmp/*.junit.xml

Gemfile

@@ -92,6 +92,7 @@ group :test do
   gem 'factory_bot'
   gem 'launchy'
   gem 'rails-controller-testing'
+  gem 'rspec_junit_formatter'
   gem 'selenium-webdriver'
   gem 'shoulda-matchers', require: false
   gem 'timecop'

Gemfile.lock

@@ -596,6 +596,8 @@ GEM
       rspec-mocks (~> 3.10)
       rspec-support (~> 3.10)
     rspec-support (3.10.2)
+    rspec_junit_formatter (0.4.1)
+      rspec-core (>= 2, < 4, != 2.12.0)
     rubocop (1.10.0)
       parallel (~> 1.10)
       parser (>= 3.0.0.0)
@@ -856,6 +858,7 @@ DEPENDENCIES
   rgeo-geojson
   rqrcode
   rspec-rails
+  rspec_junit_formatter
   rubocop
   rubocop-rails_config
   rubocop-rspec-focused