Automated E2E Testing Improvements (microsoft#1086)

* Yarn * Merge * Fixing repo * Update package * Merge * Removing change file * Transferring E2E Improvements to this branch. Other branch had weird CI errors * Change files * Upgrading WebDriverIO package * Reversing WDIO upgrade
hansenyy · Oct 20, 2021 · d4016c3 · d4016c3
1 parent fcfde38
commit d4016c3
Show file tree

Hide file tree

Showing 31 changed files with 401 additions and 105 deletions.
diff --git a/.ado/templates/e2e-publish-artifacts.yml b/.ado/templates/e2e-publish-artifacts.yml
@@ -4,17 +4,6 @@ parameters:
   buildArtifacts:
 
 steps:
-  # Generate reports. This task doesn't end until a ctrl+c is pressed. I'm not sure how to end it in this case, so
-  # I added a 1 minute timer. It should always generate the report in this time, then the task timeouts and continues on the pipeline.
-  # continueOnError is true because the pipeline should continue even on failure
-  - script: |
-      yarn report
-    workingDirectory: apps\${{parameters.platform}}
-    continueOnError: true
-    timeoutInMinutes: 1
-    displayName: "generate ${{parameters.platform}} report"
-    condition: and (succeededOrFailed(), eq(${{parameters.buildArtifacts}}, 'success'))
-
   # Copy errorShots to FluentTesterDump
   - task: CopyFiles@2
     inputs:
@@ -23,20 +12,6 @@ steps:
     displayName: "Copy tree dump screenshots"
     condition: and (succeededOrFailed(), eq(${{parameters.buildArtifacts}}, 'success'))
 
-  # Copy appium log to FluentTesterDump
-  - task: CopyFiles@2
-    inputs:
-      sourceFolder: $(Build.SourcesDirectory)\apps\${{parameters.platform}}\reports\
-      contents: "*"
-      targetFolder: $(Build.StagingDirectory)/E2E_${{parameters.applicationType}}_Dump
-    displayName: "Copy tree dump report logs"
-    condition: and (succeededOrFailed(), eq(${{parameters.buildArtifacts}}, 'success'))
-
-  - powershell: |
-      Move-Item -Path $(Build.SourcesDirectory)\apps\${{parameters.platform}}\allure-report -Destination $(Build.StagingDirectory)/E2E_${{parameters.applicationType}}_Dump
-    displayName: "Copy allure report"
-    condition: and (succeededOrFailed(), eq(${{parameters.buildArtifacts}}, 'success'))
-
   # Publish FluentTesterDump
   - task: PublishBuildArtifacts@1
     inputs:

diff --git a/apps/fluent-tester/package.json b/apps/fluent-tester/package.json
@@ -80,7 +80,7 @@
     "ts-node": "^8.10.1",
     "tsconfig-paths": "^3.9.0",
     "typescript": "3.8.3",
-    "webdriverio": "5.22.4"
+    "webdriverio": "6.2.0"
   },
   "peerDependencies": {
     "@office-iss/react-native-win32": "^0.63.7",

diff --git a/apps/fluent-tester/src/E2E/BootPage/specs/BootPages.spec.win.ts b/apps/fluent-tester/src/E2E/BootPage/specs/BootPages.spec.win.ts
@@ -25,116 +25,174 @@ const PAGE_TIMEOUT = 15000;
 describe('Open the app', function () {
   it('Boot app', () => {
     BootTestPage.waitForPageDisplayed(BOOT_APP_TIMEOUT);
+    browser.saveScreenshot('./errorShots/onBoot.png'); // Take a screenshot of the app for testing purposes
     expect(BootTestPage.isPageLoaded()).toBeTruthy();
   });
 });
 
 describe('Click on each test page and check if it renders', function () {
   it('Button Test Page', () => {
+    /* Scroll to component test page button in scrollview if not already visible*/
+    ButtonTestPage.scrollToComponentButton();
+    ButtonTestPage.waitForButtonDisplayed(PAGE_TIMEOUT);
+
+    /* Click on component button to navigate to test page */
     BootTestPage.clickAndGoToButtonPage();
     ButtonTestPage.waitForPageDisplayed(PAGE_TIMEOUT);
+
     expect(ButtonTestPage.isPageLoaded()).toBeTruthy();
   });
 
   it('Callout Test Page', () => {
+    CalloutTestPage.scrollToComponentButton();
+    CalloutTestPage.waitForButtonDisplayed(PAGE_TIMEOUT);
     BootTestPage.clickAndGoToCalloutPage();
     CalloutTestPage.waitForPageDisplayed(PAGE_TIMEOUT);
+
     expect(CalloutTestPage.isPageLoaded()).toBeTruthy();
   });
 
   it('Checkbox Test Page', () => {
+    CheckboxTestPage.scrollToComponentButton();
+    CheckboxTestPage.waitForButtonDisplayed(PAGE_TIMEOUT);
     BootTestPage.clickAndGoToCheckboxPage();
     CheckboxTestPage.waitForPageDisplayed(PAGE_TIMEOUT);
+
     expect(CheckboxTestPage.isPageLoaded()).toBeTruthy();
   });
 
   it('ContextualMenu Test Page', () => {
+    ContextualMenuTestPage.scrollToComponentButton();
+    ContextualMenuTestPage.waitForButtonDisplayed(PAGE_TIMEOUT);
     BootTestPage.clickAndGoToContextualMenuPage();
     ContextualMenuTestPage.waitForPageDisplayed(PAGE_TIMEOUT);
+
     expect(ContextualMenuTestPage.isPageLoaded()).toBeTruthy();
   });
 
   it('Experimental Tabs Test Page', () => {
+    ExperimentalTabsTestPage.scrollToComponentButton();
+    ExperimentalTabsTestPage.waitForButtonDisplayed(PAGE_TIMEOUT);
     BootTestPage.clickAndGoToExperimentalTabsPage();
     ExperimentalTabsTestPage.waitForPageDisplayed(PAGE_TIMEOUT);
+
     expect(ExperimentalTabsTestPage.isPageLoaded()).toBeTruthy();
   });
 
   it('FocusTrapZone Test Page', () => {
+    FocusTrapZoneTestPage.scrollToComponentButton();
+    FocusTrapZoneTestPage.waitForButtonDisplayed(PAGE_TIMEOUT);
     BootTestPage.clickAndGoToFocusTrapZonePage();
     FocusTrapZoneTestPage.waitForPageDisplayed(PAGE_TIMEOUT);
+
     expect(FocusTrapZoneTestPage.isPageLoaded()).toBeTruthy();
   });
 
   it('FocusZone Test Page', () => {
+    FocusZoneTestPage.scrollToComponentButton();
+    FocusZoneTestPage.waitForButtonDisplayed(PAGE_TIMEOUT);
     BootTestPage.clickAndGoToFocusZonePage();
     FocusZoneTestPage.waitForPageDisplayed(PAGE_TIMEOUT);
+
     expect(FocusZoneTestPage.isPageLoaded()).toBeTruthy();
   });
 
   it('Icon Test Page', () => {
+    IconTestPage.scrollToComponentButton();
+    IconTestPage.waitForButtonDisplayed(PAGE_TIMEOUT);
     BootTestPage.clickAndGoToIconPage();
     IconTestPage.waitForPageDisplayed(PAGE_TIMEOUT);
+
     expect(IconTestPage.isPageLoaded()).toBeTruthy();
   });
 
   it('Link Test Page', () => {
+    LinkTestPage.scrollToComponentButton();
+    LinkTestPage.waitForButtonDisplayed(PAGE_TIMEOUT);
     BootTestPage.clickAndGoToLinkPage();
     LinkTestPage.waitForPageDisplayed(PAGE_TIMEOUT);
-    expect(LinkTestPage.isPageLoaded()).toBeTruthy();
-  });
 
-  it('Tabs Test Page', () => {
-    BootTestPage.clickAndGoToTabsPage();
-    TabsTestPage.waitForPageDisplayed(PAGE_TIMEOUT);
-    expect(TabsTestPage.isPageLoaded()).toBeTruthy();
+    expect(LinkTestPage.isPageLoaded()).toBeTruthy();
   });
 
   it('Persona Test Page', () => {
+    PersonaTestPage.scrollToComponentButton();
+    PersonaTestPage.waitForButtonDisplayed(PAGE_TIMEOUT);
     BootTestPage.clickAndGoToPersonaPage();
     PersonaTestPage.waitForPageDisplayed(PAGE_TIMEOUT);
+
     expect(PersonaTestPage.isPageLoaded()).toBeTruthy();
   });
 
   it('PersonaCoin Test Page', () => {
+    PersonaCoinTestPage.scrollToComponentButton();
+    PersonaCoinTestPage.waitForButtonDisplayed(PAGE_TIMEOUT);
     BootTestPage.clickAndGoToPersonaCoinPage();
     PersonaCoinTestPage.waitForPageDisplayed(PAGE_TIMEOUT);
+
     expect(PersonaCoinTestPage.isPageLoaded()).toBeTruthy();
   });
 
   it('Pressable Test Page', () => {
+    PressableTestPage.scrollToComponentButton();
+    PressableTestPage.waitForButtonDisplayed(PAGE_TIMEOUT);
     BootTestPage.clickAndGoToPressablePage();
     PressableTestPage.waitForPageDisplayed(PAGE_TIMEOUT);
+
     expect(PressableTestPage.isPageLoaded()).toBeTruthy();
   });
 
   it('RadioGroup Test Page', () => {
+    RadioGroupTestPage.scrollToComponentButton();
+    RadioGroupTestPage.waitForButtonDisplayed(PAGE_TIMEOUT);
     BootTestPage.clickAndGoToRadioGroupPage();
     RadioGroupTestPage.waitForPageDisplayed(PAGE_TIMEOUT);
+
     expect(RadioGroupTestPage.isPageLoaded()).toBeTruthy();
   });
 
   it('Separator Test Page', () => {
+    SeparatorTestPage.scrollToComponentButton();
+    SeparatorTestPage.waitForButtonDisplayed(PAGE_TIMEOUT);
     BootTestPage.clickAndGoToSeparatorPage();
     SeparatorTestPage.waitForPageDisplayed(PAGE_TIMEOUT);
+
     expect(SeparatorTestPage.isPageLoaded()).toBeTruthy();
   });
 
   it('Svg Test Page', () => {
+    SvgTestPage.scrollToComponentButton();
+    SvgTestPage.waitForButtonDisplayed(PAGE_TIMEOUT);
     BootTestPage.clickAndGoToSvgPage();
     SvgTestPage.waitForPageDisplayed(PAGE_TIMEOUT);
+
     expect(SvgTestPage.isPageLoaded()).toBeTruthy();
   });
 
+  it('Tabs Test Page', () => {
+    TabsTestPage.scrollToComponentButton();
+    TabsTestPage.waitForButtonDisplayed(PAGE_TIMEOUT);
+    BootTestPage.clickAndGoToTabsPage();
+    TabsTestPage.waitForPageDisplayed(PAGE_TIMEOUT);
+
+    expect(TabsTestPage.isPageLoaded()).toBeTruthy();
+  });
+
   it('Text Test Page', () => {
+    TextTestPage.scrollToComponentButton();
+    TextTestPage.waitForButtonDisplayed(PAGE_TIMEOUT);
     BootTestPage.clickAndGoToTextPage();
     TextTestPage.waitForPageDisplayed(PAGE_TIMEOUT);
+
     expect(TextTestPage.isPageLoaded()).toBeTruthy();
   });
 
   it('Theme Test Page', () => {
+    ThemeTestPage.scrollToComponentButton();
+    ThemeTestPage.waitForButtonDisplayed(PAGE_TIMEOUT);
     BootTestPage.clickAndGoToThemePage();
     ThemeTestPage.waitForPageDisplayed(PAGE_TIMEOUT);
+
     expect(ThemeTestPage.isPageLoaded()).toBeTruthy();
   });
 });
diff --git a/apps/fluent-tester/src/E2E/BootPage/specs/BootTestPage.spec.uwp.ts b/apps/fluent-tester/src/E2E/BootPage/specs/BootTestPage.spec.uwp.ts
@@ -18,68 +18,102 @@ const PAGE_TIMEOUT = 15000;
 describe('Open the app', function () {
   it('Boot app', () => {
     BootTestPage.waitForPageDisplayed(BOOT_APP_TIMEOUT);
+    browser.saveScreenshot('./errorShots/onBoot.png'); // Take a screenshot of the app for testing purposes
     expect(BootTestPage.isPageLoaded()).toBeTruthy();
   });
 });
 
 describe('Click on each test page and check if it renders', function () {
   it('Button Test Page', () => {
+    /* Scroll to component test page button in scrollview if not already visible*/
+    ButtonTestPage.scrollToComponentButton();
+    ButtonTestPage.waitForButtonDisplayed(PAGE_TIMEOUT);
+
+    /* Click on component button to navigate to test page */
     BootTestPage.clickAndGoToButtonPage();
     ButtonTestPage.waitForPageDisplayed(PAGE_TIMEOUT);
+
     expect(ButtonTestPage.isPageLoaded()).toBeTruthy();
   });
 
   it('Callout Test Page', () => {
+    CalloutTestPage.scrollToComponentButton();
+    CalloutTestPage.waitForButtonDisplayed(PAGE_TIMEOUT);
     BootTestPage.clickAndGoToCalloutPage();
     CalloutTestPage.waitForPageDisplayed(PAGE_TIMEOUT);
+
     expect(CalloutTestPage.isPageLoaded()).toBeTruthy();
   });
 
   it('Checkbox Test Page', () => {
+    CheckboxTestPage.scrollToComponentButton();
+    CheckboxTestPage.waitForButtonDisplayed(PAGE_TIMEOUT);
     BootTestPage.clickAndGoToCheckboxPage();
     CheckboxTestPage.waitForPageDisplayed(PAGE_TIMEOUT);
+
     expect(CheckboxTestPage.isPageLoaded()).toBeTruthy();
   });
 
   it('Link Test Page', () => {
+    LinkTestPage.scrollToComponentButton();
+    LinkTestPage.waitForButtonDisplayed(PAGE_TIMEOUT);
     BootTestPage.clickAndGoToLinkPage();
     LinkTestPage.waitForPageDisplayed(PAGE_TIMEOUT);
+
     expect(LinkTestPage.isPageLoaded()).toBeTruthy();
   });
 
   it('PersonaCoin Test Page', () => {
+    PersonaCoinTestPage.scrollToComponentButton();
+    PersonaCoinTestPage.waitForButtonDisplayed(PAGE_TIMEOUT);
     BootTestPage.clickAndGoToPersonaCoinPage();
     PersonaCoinTestPage.waitForPageDisplayed(PAGE_TIMEOUT);
+
     expect(PersonaCoinTestPage.isPageLoaded()).toBeTruthy();
   });
 
   it('Pressable Test Page', () => {
+    PressableTestPage.scrollToComponentButton();
+    PressableTestPage.waitForButtonDisplayed(PAGE_TIMEOUT);
     BootTestPage.clickAndGoToPressablePage();
     PressableTestPage.waitForPageDisplayed(PAGE_TIMEOUT);
+
     expect(PressableTestPage.isPageLoaded()).toBeTruthy();
   });
 
   it('Separator Test Page', () => {
+    SeparatorTestPage.scrollToComponentButton();
+    SeparatorTestPage.waitForButtonDisplayed(PAGE_TIMEOUT);
     BootTestPage.clickAndGoToSeparatorPage();
     SeparatorTestPage.waitForPageDisplayed(PAGE_TIMEOUT);
-    expect(SeparatorTestPage.isPageLoaded()).toBeTruthy();
-  });
 
-  it('Text Test Page', () => {
-    BootTestPage.clickAndGoToTextPage();
-    TextTestPage.waitForPageDisplayed(PAGE_TIMEOUT);
-    expect(TextTestPage.isPageLoaded()).toBeTruthy();
+    expect(SeparatorTestPage.isPageLoaded()).toBeTruthy();
   });
 
   it('Tabs Test Page', () => {
+    TabsTestPage.scrollToComponentButton();
+    TabsTestPage.waitForButtonDisplayed(PAGE_TIMEOUT);
     BootTestPage.clickAndGoToTabsPage();
     TabsTestPage.waitForPageDisplayed(PAGE_TIMEOUT);
+
     expect(TabsTestPage.isPageLoaded()).toBeTruthy();
   });
 
+  it('Text Test Page', () => {
+    TextTestPage.scrollToComponentButton();
+    TextTestPage.waitForButtonDisplayed(PAGE_TIMEOUT);
+    BootTestPage.clickAndGoToTextPage();
+    TextTestPage.waitForPageDisplayed(PAGE_TIMEOUT);
+
+    expect(TextTestPage.isPageLoaded()).toBeTruthy();
+  });
+
   it('Theme Test Page', () => {
+    ThemeTestPage.scrollToComponentButton();
+    ThemeTestPage.waitForButtonDisplayed(PAGE_TIMEOUT);
     BootTestPage.clickAndGoToThemePage();
     ThemeTestPage.waitForPageDisplayed(PAGE_TIMEOUT);
+
     expect(ThemeTestPage.isPageLoaded()).toBeTruthy();
   });
 });
diff --git a/apps/fluent-tester/src/E2E/Button/pages/ButtonTestPage.win.ts b/apps/fluent-tester/src/E2E/Button/pages/ButtonTestPage.win.ts
@@ -1,4 +1,4 @@
-import { BUTTON_TESTPAGE } from '../../../FluentTester/TestComponents/Button/consts';
+import { BUTTON_TESTPAGE, HOMEPAGE_BUTTON_BUTTON } from '../../../FluentTester/TestComponents/Button/consts';
 import { BasePage, By } from '../../common/BasePage';
 
 class ButtonTestPage extends BasePage {
@@ -9,6 +9,10 @@ class ButtonTestPage extends BasePage {
   get _pageName() {
     return BUTTON_TESTPAGE;
   }
+
+  get _pageButton() {
+    return By(HOMEPAGE_BUTTON_BUTTON);
+  }
 }
 
 export default new ButtonTestPage();
diff --git a/apps/fluent-tester/src/E2E/Callout/pages/CalloutTestPage.win.ts b/apps/fluent-tester/src/E2E/Callout/pages/CalloutTestPage.win.ts
@@ -1,4 +1,4 @@
-import { CALLOUT_TESTPAGE } from '../../../FluentTester/TestComponents/Callout/consts';
+import { CALLOUT_TESTPAGE, HOMEPAGE_CALLOUT_BUTTON } from '../../../FluentTester/TestComponents/Callout/consts';
 import { BasePage, By } from '../../common/BasePage';
 
 class CalloutTestPage extends BasePage {
@@ -9,6 +9,10 @@ class CalloutTestPage extends BasePage {
   get _pageName() {
     return CALLOUT_TESTPAGE;
   }
+
+  get _pageButton() {
+    return By(HOMEPAGE_CALLOUT_BUTTON);
+  }
 }
 
 export default new CalloutTestPage();
diff --git a/apps/fluent-tester/src/E2E/Checkbox/pages/CheckboxTestPage.win.ts b/apps/fluent-tester/src/E2E/Checkbox/pages/CheckboxTestPage.win.ts
@@ -1,4 +1,4 @@
-import { CHECKBOX_TESTPAGE } from '../../../FluentTester/TestComponents/Checkbox/consts';
+import { CHECKBOX_TESTPAGE, HOMEPAGE_CHECKBOX_BUTTON } from '../../../FluentTester/TestComponents/Checkbox/consts';
 import { BasePage, By } from '../../common/BasePage';
 
 class CheckboxTestPage extends BasePage {
@@ -9,6 +9,10 @@ class CheckboxTestPage extends BasePage {
   get _pageName() {
     return CHECKBOX_TESTPAGE;
   }
+
+  get _pageButton() {
+    return By(HOMEPAGE_CHECKBOX_BUTTON);
+  }
 }
 
 export default new CheckboxTestPage();