Add playwright tests, refs #2635

Merge request studip/studip!1790

.gitignore

 *~
 
+.env
+
 .webpack.*
 composer
 node_modules
@@ -40,6 +42,9 @@ tests/_helpers/TestGuy.php
 tests/_helpers/WebGuy.php
 tests/_helpers/_generated
 tests/_output/
+tests/e2e/.auth
+tests/e2e/test-results/*
+playwright-report/*
 
 .idea
 /config/oauth2/*.key

.gitlab-ci.yml

 image: studip/studip:tests-php7.2
 
 variables:
+  FF_NETWORK_PER_BUILD: 1
   GIT_DEPTH: 1
   MYSQL_RANDOM_ROOT_PASSWORD: "true"
   MYSQL_DATABASE: studip_db
@@ -304,6 +305,55 @@ test-assets:
   script:
     - npm run webpack-dev
 
+test-e2e:
+  stage: test
+  #  needs: [lint-css, lint-js, lint-php]
+  image: mcr.microsoft.com/playwright:v1.33.0-jammy
+  services:
+    - mariadb
+  variables:
+    PHP_WEBSERVER_URL: localhost:65432
+    E2E_REPORT: $REPORT_DIR/e2e.xml
+  interruptible: true
+  when: manual
+  cache:
+    - *composer-cache
+    - *npm-cache
+  before_script:
+    - mkdir ./bin
+    - apt-get update
+    - apt -y install software-properties-common
+    - add-apt-repository ppa:ondrej/php
+    - apt-get update
+    - DEBIAN_FRONTEND=noninteractive
+      apt-get -yq install
+      make zip unzip mariadb-client
+      php7.4 libapache2-mod-php7.4 php7.4-common php7.4-curl php7.4-mbstring
+      php7.4-xmlrpc php7.4-mysql php7.4-gd php7.4-xml php7.4-intl php7.4-ldap
+      php7.4-imagick php7.4-json php7.4-cli
+    - echo "short_open_tag=On" >> /etc/php/7.4/php.ini
+    - echo "short_open_tag=On" >> /etc/php/7.4/cli/php.ini
+    - php -r "copy('https://getcomposer.org/installer', 'composer-setup.php');"
+    - php composer-setup.php  --install-dir=./bin --filename=composer
+    - export PATH="./bin:$PATH"
+    - php -r "unlink('composer-setup.php');"
+    - *mkdir-reports
+    - *initialize-studip-database
+    - ./cli/studip config:set SHOW_TERMS_ON_FIRST_LOGIN 0
+    - npm install playwright
+    - npm ci
+    - npx playwright install --with-deps
+  script:
+    - php -S $PHP_WEBSERVER_URL -t public -q &
+    - PHP_SERVER_PID=$!
+    - PLAYWRIGHT_JUNIT_OUTPUT_NAME="$E2E_REPORT"
+      PLAYWRIGHT_BASE_URL="http://$PHP_WEBSERVER_URL"
+      npx playwright test --reporter=junit --grep-invert a11y
+    - kill -3 $PHP_SERVER_PID
+  artifacts:
+    reports:
+      junit: $E2E_REPORT
+
 packaging:
   stage: packaging
   cache: []

Makefile

@@ -72,6 +72,9 @@ test-functional: $(CODECEPT)
 test-jsonapi: $(CODECEPT)
 	$(CODECEPT) run jsonapi
 
+test-e2e: npm
+	npx playwright test
+
 test-unit: $(CODECEPT)
 	$(CODECEPT) run unit
 

package.json

@@ -19,6 +19,7 @@
         "node": ">=16"
     },
     "devDependencies": {
+        "@axe-core/playwright": "^4.6.1",
         "@babel/core": "^7.17.9",
         "@babel/eslint-parser": "^7.17.0",
         "@babel/plugin-syntax-dynamic-import": "^7.8.3",
@@ -57,6 +58,7 @@
         "@fullcalendar/resource-timegrid": "^4.3.0",
         "@fullcalendar/resource-timeline": "^4.3.0",
         "@fullcalendar/timegrid": "^4.3.0",
+        "@playwright/test": "^1.33.0",
         "@popperjs/core": "^2.11.2",
         "@types/jquery": "^3.5.16",
         "@types/jqueryui": "^1.12.16",
@@ -73,6 +75,7 @@
         "cropperjs": "1.5.9",
         "css-loader": "^5.0.1",
         "css-minimizer-webpack-plugin": "^1.1.5",
+        "dotenv": "^16.0.3",
         "easygettext": "^2.17.0",
         "es6-promise": "4.2.8",
         "eslint": "^7.32.0",

playwright.config.ts

+import { defineConfig, devices } from '@playwright/test';
+import dotenv from 'dotenv';
+import path from 'path';
+
+
+// Read from default ".env" file.
+dotenv.config();
+
+/**
+ * See https://playwright.dev/docs/test-configuration.
+ */
+export default defineConfig({
+    testDir: path.resolve(__dirname, 'tests', 'e2e'),
+    outputDir: path.resolve(__dirname, 'tests', 'e2e', 'test-results'),
+    /* Run tests in files in parallel */
+    fullyParallel: true,
+    /* Fail the build on CI if you accidentally left test.only in the source code. */
+    forbidOnly: !!process.env.CI,
+    /* Retry on CI only */
+    retries: process.env.CI ? 3 : 0,
+    /* Opt out of parallel tests on CI. */
+    workers: process.env.CI ? 1 : undefined,
+    /* Reporter to use. See https://playwright.dev/docs/test-reporters */
+    reporter: 'html',
+    /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
+    use: {
+        /* Base URL to use in actions like `await page.goto('/')`. */
+        baseURL: process.env.PLAYWRIGHT_BASE_URL ?? 'http://127.0.0.1',
+
+        locale: process.env.PLAYWRIGHT_LOCALE ?? 'de_DE',
+
+        /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
+        trace: 'on-first-retry',
+
+        actionTimeout: 10 * 1000,
+        navigationTimeout: 30 * 1000,
+
+        launchOptions: {
+            slowMo: process.env.PLAYWRIGHT_TEST_SPEED ?? 50,
+        },
+    },
+    expect: {
+        timeout: 10 * 1000,
+    },
+
+    /* Configure projects for major browsers */
+    projects: [
+
+        { name: 'setup', testMatch: /.*\.setup\.ts/ },
+
+        // {
+        //     name: 'chromium',
+        //     use: { ...devices['Desktop Chrome'] },
+        // },
+
+        {
+            name: 'firefox',
+            use: { ...devices['Desktop Firefox'] },
+            dependencies: ['setup'],
+        },
+
+        // {
+        //     name: 'webkit',
+        //     use: { ...devices['Desktop Safari'] },
+        // },
+
+        /* Test against mobile viewports. */
+        // {
+        //   name: 'Mobile Chrome',
+        //   use: { ...devices['Pixel 5'] },
+        // },
+        // {
+        //   name: 'Mobile Safari',
+        //   use: { ...devices['iPhone 12'] },
+        // },
+
+        /* Test against branded browsers. */
+        // {
+        //   name: 'Microsoft Edge',
+        //   use: { ...devices['Desktop Edge'], channel: 'msedge' },
+        // },
+        // {
+        //   name: 'Google Chrome',
+        //   use: { ..devices['Desktop Chrome'], channel: 'chrome' },
+        // },
+    ],
+});

tests/e2e/a11y.spec.ts

+import { test, expect } from '@playwright/test';
+import AxeBuilder from '@axe-core/playwright';
+
+test.describe('Should not have any automatically detectable accessibility issues @a11y', () => {
+    test.describe('while not logged in', () => {
+        const pages = [
+            ['on homepage', 'index.php?cancel_login=1'],
+            ['on login page', 'index.php?again=yes'],
+            ['on password forgotten page', 'dispatch.php/new_password?cancel_login=1'],
+        ];
+
+        pages.forEach(async ([title, url]) =>
+            test(title, async ({ page, baseURL }) => {
+                await page.goto(url);
+                const accessibilityScanResults = await analyze(page);
+                expect(accessibilityScanResults.violations).toEqual([]);
+            })
+        );
+    });
+
+    test.describe('while logged in as author', () => {
+        const autorFile = 'tests/e2e/.auth/autor.json';
+        test.use({ storageState: autorFile });
+
+        const pages = [
+            ['on start page', 'dispatch.php/start'],
+            ['on profile page', 'dispatch.php/profile/index'],
+            ['on my courses page', 'dispatch.php/courses'],
+        ];
+
+        pages.forEach(async ([title, url]) =>
+            test(title, async ({ page, baseURL }) => {
+                await page.goto(url);
+                const accessibilityScanResults = await analyze(page);
+                expect(accessibilityScanResults.violations).toEqual([]);
+            })
+        );
+    });
+});
+
+function analyze(page) {
+    return new AxeBuilder({ page }).analyze();
+}

tests/e2e/auth.setup.ts

+import { test as setup, expect } from '@playwright/test';
+import { credentials } from './credentials.ts';
+
+const rootFile = 'tests/e2e/.auth/root.json';
+const adminFile = 'tests/e2e/.auth/admin.json';
+const dozentFile = 'tests/e2e/.auth/dozent.json';
+const tutorFile = 'tests/e2e/.auth/tutor.json';
+const autorFile = 'tests/e2e/.auth/autor.json';
+
+setup('authenticate as root', async ({ page }) => {
+    await login(page, credentials.root);
+    await page.context().storageState({ path: rootFile });
+});
+
+setup('authenticate as admin', async ({ page }) => {
+    await login(page, credentials.admin);
+    await page.context().storageState({ path: adminFile });
+});
+
+setup('authenticate as dozent', async ({ page }) => {
+    await login(page, credentials.dozent);
+    await page.context().storageState({ path: dozentFile });
+});
+
+setup('authenticate as tutor', async ({ page }) => {
+    await login(page, credentials.tutor);
+    await page.context().storageState({ path: tutorFile });
+});
+
+setup('authenticate as autor', async ({ page }) => {
+    await login(page, credentials.autor);
+    await page.context().storageState({ path: autorFile });
+});
+
+async function login(page, { username, password }) {
+    await page.goto('index.php?again=yes');
+
+    await page.getByLabel(/Benutzername/i).fill(username);
+    await page.getByLabel(/Passwort/i).fill(password);
+    await page.getByRole('button', { name: 'Anmelden' }).click();
+    await expect(page.locator('#avatar-menu-container')).toBeVisible();
+}

tests/e2e/credentials.ts

+export const credentials = {
+    root: {
+        username: process.env.PLAYWRIGHT_CREDENTIALS_ROOT_USERNAME ?? 'root@studip',
+        password: process.env.PLAYWRIGHT_CREDENTIALS_ROOT_PASSWORD ?? 'testing',
+    },
+    admin: {
+        username: process.env.PLAYWRIGHT_CREDENTIALS_ADMIN_USERNAME ?? 'test_admin',
+        password: process.env.PLAYWRIGHT_CREDENTIALS_ADMIN_PASSWORD ?? 'testing',
+    },
+    dozent: {
+        username: process.env.PLAYWRIGHT_CREDENTIALS_DOZENT_USERNAME ?? 'test_dozent',
+        password: process.env.PLAYWRIGHT_CREDENTIALS_DOZENT_PASSWORD ?? 'testing',
+    },
+    tutor: {
+        username: process.env.PLAYWRIGHT_CREDENTIALS_TUTOR_USERNAME ?? 'test_tutor',
+        password: process.env.PLAYWRIGHT_CREDENTIALS_TUTOR_PASSWORD ?? 'testing',
+    },
+    autor: {
+        username: process.env.PLAYWRIGHT_CREDENTIALS_AUTOR_USERNAME ?? 'test_autor',
+        password: process.env.PLAYWRIGHT_CREDENTIALS_AUTOR_PASSWORD ?? 'testing',
+    },
+};

tests/e2e/login.spec.ts

+import { test, expect } from '@playwright/test';
+import { credentials } from './credentials.ts';
+
+test.describe('Loggin In - HTML Web Form @auth', () => {
+    test.describe('Coming from homepage', () => {
+        test('should take us to the login form @smoke', async ({ page, baseURL }) => {
+            await page.goto('');
+
+            await expect(page.locator('#loginbox')).toBeVisible();
+
+            const loginLink = page.getByRole('link', { name: 'Login für registrierte NutzerInnen' });
+            await expect(loginLink).toBeVisible();
+            await loginLink.click();
+
+            const benutzername = page.getByLabel(/Benutzername/i);
+            await expect(benutzername).toBeVisible();
+            await expect(benutzername).toBeEditable();
+
+            const passwort = page.getByLabel(/Passwort/i);
+            await expect(passwort).toBeVisible();
+            await expect(passwort).toBeEditable();
+        });
+    });
+
+    test.describe('Unauthorized', () => {
+        test('redirects to the login form @smoke', async ({ page }) => {
+            await page.goto('dispatch.php/start');
+
+            await expect(page.getByLabel(/Passwort/)).toBeVisible();
+            await expect(page.locator('#avatar-menu-container')).not.toBeVisible();
+        });
+    });
+
+    test.describe('HTML Form submission', () => {
+        test.beforeEach(async ({ page }) => {
+            await page.goto('index.php?again=yes');
+        });
+
+        test('displays error on invalid login', async ({ page }) => {
+            const benutzername = page.getByLabel(/Benutzername/i);
+            const passwort = page.getByLabel(/Passwort/i);
+            const submit = page.getByRole('button', { name: 'Anmelden' });
+
+            await benutzername.fill('username');
+            await passwort.fill('password');
+            await submit.click();
+            await expect(page.locator('css=.messagebox_error')).toBeVisible();
+        });
+
+        test('redirects to start page', async ({ page, baseURL }) => {
+            const benutzername = page.getByLabel(/Benutzername/i);
+            const passwort = page.getByLabel(/Passwort/i);
+            const submit = page.getByRole('button', { name: 'Anmelden' });
+
+            await benutzername.fill(credentials.autor.username);
+            await passwort.fill(credentials.autor.password);
+            await submit.click();
+            await expect(page.locator('#avatar-menu-container')).toBeVisible();
+            await expect(page).toHaveURL(`${baseURL}dispatch.php/start`);
+        });
+    });
+});

tests/e2e/logout.spec.ts

+import { test, expect } from '@playwright/test';
+
+const dozentFile = 'tests/e2e/.auth/dozent.json';
+
+test.describe('Logging Out', () => {
+    test.use({ storageState: dozentFile });
+
+    test('should take us back to the homepage', async ({ page, baseURL }) => {
+        await page.goto(baseURL);
+        await expect(page.locator('#avatar-menu-container')).toBeVisible();
+        await page.getByTitle('Testaccount Dozent').click();
+        await page.getByRole('link', { name: 'Logout' }).click();
+        await expect(page).toHaveURL(/index\.php.*logout=true/);
+    });
+});

tests/e2e/plugin-administration.spec.js

+import { test, expect } from '@playwright/test';
+
+const rootFile = 'tests/e2e/.auth/root.json';
+
+test.describe('Visiting the plugin administration @root @plugins', () => {
+    test.use({ storageState: rootFile });
+
+    test('should let us deactivate and re-activate plugins', async ({ page }) => {
+        await page.goto('dispatch.php/admin/plugin');
+        const tableRow = await page.getByRole('row', { name: /TerminWidget/i });
+        const input = await tableRow.locator('input[name*=enabled]');
+        const saveButton = await page.getByRole('button', { name: 'Speichern' });
+
+        await input.uncheck();
+        await saveButton.click();
+        await expect(page.getByText('Plugin "TerminWidget" wurde deaktiviert')).toBeVisible();
+
+        await input.check();
+        await saveButton.click();
+        await expect(page.getByText('Plugin "TerminWidget" wurde aktiviert')).toBeVisible();
+    });
+});

tests/e2e/user-courseware.spec.js

+import { test, expect } from '@playwright/test';
+
+const autorFile = 'tests/e2e/.auth/autor.json';
+
+test.describe('Visiting my Arbeitsplatz @autor @courseware', () => {
+    test.use({ storageState: autorFile });
+
+    test('should let us create a new Lernmaterial', async ({ page }) => {
+        await page.goto('dispatch.php/start');
+        await page.getByRole('link', { name: /Courseware Erstellen/ }).click();
+        await page.getByRole('button', { name: 'Lernmaterial hinzufügen' }).click();
+        await page.getByLabel('Titel des Lernmaterials*').click();
+        await page.getByLabel('Titel des Lernmaterials*').fill('Ein Titel');
+        await page.getByLabel('Titel des Lernmaterials*').press('Tab');
+        await page.getByLabel('Beschreibung*').fill('Eine Beschreibung');
+        await page.getByRole('tab', { name: 'Erscheinung' }).click();
+        await page
+            .getByRole('combobox', { name: 'Search for option' })
+            .locator('div')
+            .filter({ hasText: 'Blau' })
+            .click();
+        await page.getByRole('button', { name: 'Erstellen' }).click();
+        const lernmaterial = await page.getByRole('link', { name: 'Ein Titel Eine Beschreibung' }).last();
+        await expect(lernmaterial).toBeVisible();
+    });
+});