Fix linux autofill CI failures (#378)

* lengthen timeouts for xfail tests in autofill

* restrict tc runs

* find error details

* fill with different function

* try screenshots

* moar screenshots

* moar screenshots

* add one sleep

* wait for confirmation on cc save

* wait for popup to go away

* try a hard wait when filling out autofill forms

* detach from autofill class methods

* skip unnecessary screenshots

* more screenshots mid flight

* defactor some more

* get the source

* wait for chrome?

* write chrome to artifacts

* wait for chrome context, then log

* hide autofill menu?

* try a new tab?

* try headed

* try with wayland mode disabled

* try with wayland mode disabled

* try with wayland mode disabled

* try with wayland mode explicitly enabled

* try wayland-specific stuff

* lint

* wayland enable + headed

* get documentURI

* try 2404 wayland the bad way

* try 2404 wayland the bad way

* try 2404 wayland the hard way

* don't default to pynput import

* don't default to pynput import

* try headless

* try a different way of clicking save

* revert everything but the selector

* try to fix fullscreen shot on linux

* try to fix fullscreen shot on linux 2

* fix linux screenshot

* fix with no-save pref

* fix with CI setup

* try new unlock logic

* try bypassing keyring with env vars

* try on relsre image

* try on relsre image

* merge main and rerun on relsre image

* force autofill rerun

* prove that keyring is still the problem

* get pids

* write pids to file

* try to actually set a password

* try to actually set a password

* kill keyring daemon and restart with silly password

* new meaningless change

* what is in the CI list?

* fix choose script again

* remove xfails

* doc

.github/workflows/smoke.yml

@@ -177,6 +177,7 @@ jobs:
           FX_EXECUTABLE: /Volumes/Firefox/Firefox.app/Contents/MacOS/firefox
         run: |
           /Volumes/Firefox/Firefox.app/Contents/MacOS/firefox --version
+          pipenv run python choose_ci_set.py
           pipenv run pytest --fx-executable="$FX_EXECUTABLE" -n 4 $(python choose_ci_set.py) || TEST_EXIT_CODE=$?
           mv artifacts artifacts-mac || true
           exit $TEST_EXIT_CODE

choose_ci_set.py

@@ -59,6 +59,27 @@ def get_tests_by_model(
     return matching_tests
 
 
+def dedupe(run_list: list, slash: str) -> list:
+    """For a run list, remove entries that are covered by more general entries."""
+    suites = []
+    run_list = list(set(run_list))
+    for entry in run_list:
+        pieces = entry.split(slash)
+        if len(pieces) < 4:
+            suites.append(pieces[-1])
+
+    removes = []
+    for entry in run_list:
+        pieces = entry.split(slash)
+        if len(pieces) > 3 and pieces[2] in suites:
+            removes.append(run_list.index(entry))
+
+    for remove in removes:
+        del run_list[remove]
+
+    return run_list
+
+
 if __name__ == "__main__":
     if os.path.exists(".env"):
         with open(".env") as fh:
@@ -134,7 +155,7 @@ def get_tests_by_model(
 
     if changed_suite_conftests:
         run_list = [
-            os.path.join(*suite.split(slash)[-3:-1])
+            "." + slash + os.path.join(*suite.split(slash)[-3:-1])
             for suite in changed_suite_conftests
         ]
 
@@ -161,6 +182,12 @@ def get_tests_by_model(
         for changed_test in changed_tests:
             found = False
             for file in run_list:
+                # Don't add if already exists in suite changes
+                pieces = file.split(slash)
+                if len(pieces) == 3 and pieces[-1] in changed_test:
+                    found = True
+
+                # Don't add if already in list
                 if file in changed_test:
                     found = True
             if not found:
@@ -172,5 +199,5 @@ def get_tests_by_model(
         run_list.extend(ci_paths)
 
         # Dedupe just in case
-        run_list = list(set(run_list))
+        run_list = dedupe(run_list, slash)
         print("\n".join(run_list))

taskcluster/config.yml

@@ -30,4 +30,4 @@ workers:
       provisioner: '{trust-domain}-t'
       implementation: generic-worker
       os: linux
-      worker-type: 't-linux-2204-wayland'
+      worker-type: 't-linux-2204-wayland-relsre'

taskcluster/kinds/run-smoke-tests/kind.yml

@@ -33,6 +33,9 @@ tasks:
         pipenv install;
         ./collect_executables.sh;
         ./firefox/firefox --version;
+        # auth into keyring
+        killall -q -u "$(whoami)" gnome-keyring-daemon;
+        echo -n 'M0z1ll4!' | gnome-keyring-daemon --daemonize --login | sed -e 's/^/export'
         pipenv run pytest --fx-executable ./firefox/firefox -n 4 $(python3 choose_ci_set.py)
         export FAILURE=${?}
         mv ./ci_pyproject_headed.toml ./pyproject.toml;

tests/form_autofill/conftest.py

@@ -1,5 +1,7 @@
 import pytest
 
+# new meaningless change, once again
+
 
 @pytest.fixture()
 def suite_id():

tests/form_autofill/test_autofill_cc_cvv.py

@@ -1,6 +1,5 @@
 import json
 import logging
-import platform
 
 import pytest
 from selenium.webdriver import Firefox
@@ -16,7 +15,6 @@ def test_case():
     return "122399"
 
 
-@pytest.mark.xfail(platform.system() == "Linux", reason="Autofill Linux instability")
 def test_autofill_cc_cvv(driver: Firefox):
     """
     C122399, Test form autofill CC CVV number

tests/form_autofill/test_autofill_credit_card.py

@@ -1,5 +1,3 @@
-import platform
-
 import pytest
 from selenium.webdriver import Firefox
 
@@ -13,7 +11,6 @@ def test_case():
     return "122405"
 
 
-@pytest.mark.xfail(platform.system() == "Linux", reason="Autofill Linux instability")
 def test_autofill_credit_card(driver: Firefox):
     """
     C122405, tests that after filling autofill and disabling cc info it appears in panel

tests/form_autofill/test_autofill_credit_card_four_fields.py

@@ -1,5 +1,3 @@
-import platform
-
 import pytest
 from selenium.webdriver import Firefox
 
@@ -13,7 +11,6 @@ def test_case():
     return "122404"
 
 
-@pytest.mark.xfail(platform.system() == "Linux", reason="Autofill Linux instability")
 def test_autofill_four_fields(driver: Firefox):
     """
     C122404, tests that the form fields are filled corrected after saving a profile.

tests/form_autofill/test_cc_clear_form.py

@@ -1,5 +1,3 @@
-import platform
-
 import pytest
 from selenium.webdriver import Firefox
 
@@ -13,7 +11,6 @@ def test_case():
     return "122581"
 
 
-@pytest.mark.xfail(platform.system() == "Linux", reason="Autofill Linux instability")
 def test_clear_form_credit_card(driver: Firefox):
     """
     C122581, Test clear form credit card

tests/form_autofill/test_clear_form.py

@@ -1,5 +1,3 @@
-import platform
-
 import pytest
 from selenium.webdriver import Firefox
 
@@ -16,7 +14,6 @@ def test_case():
 countries = ["CA", "US"]
 
 
-@pytest.mark.xfail(platform.system() == "Linux", reason="Autofill Linux instability")
 @pytest.mark.parametrize("country_code", countries)
 def test_clear_form(driver: Firefox, country_code: str):
     """

tests/form_autofill/test_create_new_cc_profile.py

@@ -1,5 +1,4 @@
 import json
-import platform
 
 import pytest
 from selenium.webdriver import Firefox
@@ -13,7 +12,6 @@ def test_case():
     return "122389"
 
 
-@pytest.mark.xfail(platform.system() == "Linux", reason="Autofill Linux instability")
 def test_create_new_cc_profile(driver: Firefox):
     """
     C122389, tests you can create and save a new Credit Card profile

tests/form_autofill/test_delete_cc_profile.py

@@ -1,5 +1,3 @@
-import platform
-
 import pytest
 from selenium.webdriver import Firefox
 
@@ -14,7 +12,6 @@ def test_case():
     return "122391"
 
 
-@pytest.mark.xfail(platform.system() == "Linux", reason="Autofill Linux instability")
 def test_delete_cc_profile(driver: Firefox):
     """
     C122391, Ensuring that deleting cc profiles will make it so CC does not show up in the grid

tests/form_autofill/test_edit_credit_card.py

@@ -1,6 +1,5 @@
 import json
 import logging
-import platform
 
 import pytest
 from selenium.webdriver import Firefox
@@ -26,7 +25,6 @@ def hard_quit():
 tabs = [i for i in range(4)]
 
 
-@pytest.mark.xfail(platform.system() == "Linux", reason="Autofill Linux instability")
 @pytest.mark.parametrize("num_tabs", tabs)
 def test_edit_credit_card_profile(driver: Firefox, num_tabs: int, hard_quit):
     """

tests/form_autofill/test_form_autofill_suggestions.py

@@ -1,5 +1,3 @@
-import platform
-
 import pytest
 from selenium.webdriver import Firefox
 
@@ -16,7 +14,6 @@ def test_case():
 indices = range(2)
 
 
-@pytest.mark.xfail(platform.system() == "Linux", reason="Autofill Linux instability")
 @pytest.mark.parametrize("index", indices)
 def test_form_autofill_suggestions(driver: Firefox, index: str):
     """

tests/form_autofill/test_updating_credit_card.py

@@ -1,6 +1,5 @@
 import json
 import logging
-import platform
 
 import pytest
 from selenium.webdriver import Firefox
@@ -19,7 +18,6 @@ def test_case():
 fields = ["cc-name", "cc-exp-month", "cc-exp-year"]
 
 
-@pytest.mark.xfail(platform.system() == "Linux", reason="Autofill Linux instability")
 @pytest.mark.parametrize("field", fields)
 def test_update_cc_no_dupe_name(driver: Firefox, field: str):
     """
@@ -69,7 +67,6 @@ def test_update_cc_no_dupe_name(driver: Firefox, field: str):
     about_prefs.verify_cc_json(cc_info_json, credit_card_sample_data)
 
 
-@pytest.mark.xfail(platform.system() == "Linux", reason="Autofill Linux instability")
 def test_update_cc_number_new_profile(driver: Firefox):
     """
     C122406, continuation ensures that updating the credit card number saves a new card instead of updating the new one