diff --git a/tests/e2e/tests/e2e/GitSsh.spec.ts b/tests/e2e/tests/e2e/GitSsh.spec.ts index 6daf1065ad..8095d452f0 100644 --- a/tests/e2e/tests/e2e/GitSsh.spec.ts +++ b/tests/e2e/tests/e2e/GitSsh.spec.ts @@ -44,7 +44,7 @@ const testWorkspaceUtils: TestWorkspaceUtil = e2eContainer.get { - const workspacePrefixUrl: string = `${TestConstants.TS_SELENIUM_BASE_URL}/dashboard/#/ide/TestConstants.TS_SELENIUM_USERNAME/`; + const workspacePrefixUrl: string = `${TestConstants.TS_SELENIUM_BASE_URL}/dashboard/#/ide/${TestConstants.TS_SELENIUM_USERNAME}/`; const wsNameCheckGeneratingKeys = 'checkGeneraringSsh'; const wsNameCheckPropagatingKeys = 'checkPropagatingSsh'; const committedFile = 'README.md'; diff --git a/tests/e2e/tests/load_test/LoadTest.spec.ts b/tests/e2e/tests/load_test/LoadTest.spec.ts index 0b2b6443fa..08d08f522c 100644 --- a/tests/e2e/tests/load_test/LoadTest.spec.ts +++ b/tests/e2e/tests/load_test/LoadTest.spec.ts @@ -15,27 +15,34 @@ import { ProjectTree } from '../../pageobjects/ide/ProjectTree'; import { DriverHelper } from '../../utils/DriverHelper'; import { ICheLoginPage } from '../../pageobjects/login/ICheLoginPage'; import { TestWorkspaceUtil } from '../../utils/workspace/TestWorkspaceUtil'; -import { TestConstants } from '../..'; +import { TestConstants, NameGenerator } from '../..'; const driverHelper: DriverHelper = e2eContainer.get(CLASSES.DriverHelper); const ide: Ide = e2eContainer.get(CLASSES.Ide); const projectTree: ProjectTree = e2eContainer.get(CLASSES.ProjectTree); -const factoryUrl: string = `${TestConstants.TS_SELENIUM_BASE_URL}/f?url=https://gist.githubusercontent.com/Katka92/fe747fa93b8275abb2fb9f3803de1f0f/raw`; const cheLoginPage: ICheLoginPage = e2eContainer.get(TYPES.CheLogin); const testWorkspaceUtils: TestWorkspaceUtil = e2eContainer.get(TYPES.WorkspaceUtil); +const workspaceName: string = NameGenerator.generate('wksp-test-', 5); +const workspacePrefixUrl: string = `${TestConstants.TS_SELENIUM_BASE_URL}/dashboard/#/ide/${TestConstants.TS_SELENIUM_USERNAME}/`; +const namespace: string = TestConstants.TS_SELENIUM_USERNAME; suite('Load test suite', async () => { suiteTeardown (async function () { await testWorkspaceUtils.cleanUpAllWorkspaces(); }); - test('Login and navigate to factory url', async () => { - await driverHelper.navigateToUrl(factoryUrl); + suiteSetup(async function () { + const wsConfig = await testWorkspaceUtils.getBaseDevfile(); + wsConfig.metadata!.name = workspaceName; + await testWorkspaceUtils.createWsFromDevFile(wsConfig); + }); + + test('Login into workspace and open tree container', async () => { + await driverHelper.navigateToUrl(workspacePrefixUrl + workspaceName); await cheLoginPage.login(); }); + test('Wait loading workspace and get time', async () => { - console.log('Waiting for workspace to start.'); - await ide.waitAndSwitchToIdeFrame(); - console.log('Waiting for project to be imported and opened.'); + await ide.waitWorkspaceAndIde(namespace, workspaceName); await projectTree.openProjectTreeContainer(); }); diff --git a/tests/performance/create_users.sh b/tests/performance/create_users.sh index c7808c648d..9a07cfb9b3 100755 --- a/tests/performance/create_users.sh +++ b/tests/performance/create_users.sh @@ -46,7 +46,7 @@ done # check that all variables are set -if [ -z $USERNAME ] || [ -z $STARTING_INDEX ] || [ -z $USER_COUNT ] || [ -z $PASSWORD ] || [ -z $TESTED_ENV]; then +if [ -z $USERNAME ] || [ -z $STARTING_INDEX ] || [ -z $USER_COUNT ] || [ -z $PASSWORD ] || [ -z $TESTED_ENV ]; then echo "ERROR: All variables must be set. See the text below to learn how to use this script:" printHelp exit 1 @@ -66,11 +66,11 @@ else fi KEYCLOAK_URL=$(oc get route/keycloak -o jsonpath='{.spec.host}') -KEYCLOAK_BASE_URL="http://${KEYCLOAK_URL}/auth" +KEYCLOAK_BASE_URL="https://${KEYCLOAK_URL}/auth" # get admin token echo "Getting admin token" -ADMIN_ACCESS_TOKEN=$(curl -X POST -H "Content-Type: application/x-www-form-urlencoded" -d "username=${ADMIN_USERNAME}" -d "password=${ADMIN_PASS}" -d "grant_type=password" -d "client_id=admin-cli" $KEYCLOAK_BASE_URL/realms/master/protocol/openid-connect/token | jq -r .access_token) +ADMIN_ACCESS_TOKEN=$(curl -X POST -k -H "Content-Type: application/x-www-form-urlencoded" -d "username=${ADMIN_USERNAME}" -d "password=${ADMIN_PASS}" -d "grant_type=password" -d "client_id=admin-cli" $KEYCLOAK_BASE_URL/realms/master/protocol/openid-connect/token | jq -r .access_token) if [ -z $ADMIN_ACCESS_TOKEN ]; then echo "ERROR: Could not obtain admin access token." @@ -86,12 +86,12 @@ do USER_JSON={\"username\":\"${TEST_USERNAME}\",\"enabled\":true,\"emailVerified\":true,\"email\":\"test${user_number}@user.aa\"} echo "Creating user" - curl -X POST $KEYCLOAK_BASE_URL/admin/realms/$REALM/users -H "Authorization: Bearer ${ADMIN_ACCESS_TOKEN}" -H "Content-Type: application/json" -d "${USER_JSON}" - USER_ID=$(curl -X GET $KEYCLOAK_BASE_URL/admin/realms/$REALM/users?username=${TEST_USERNAME} -H "Authorization: Bearer ${ADMIN_ACCESS_TOKEN}" | jq -r .[0].id) + curl -X POST -k $KEYCLOAK_BASE_URL/admin/realms/$REALM/users -H "Authorization: Bearer ${ADMIN_ACCESS_TOKEN}" -H "Content-Type: application/json" -d "${USER_JSON}" + USER_ID=$(curl -X GET -k $KEYCLOAK_BASE_URL/admin/realms/$REALM/users?username=${TEST_USERNAME} -H "Authorization: Bearer ${ADMIN_ACCESS_TOKEN}" | jq -r .[0].id) echo "User id: $USER_ID" echo "Updating password" CREDENTIALS_JSON="{\"type\":\"password\",\"value\":\"${PASSWORD}\",\"temporary\":false}" - curl -X PUT $KEYCLOAK_BASE_URL/admin/realms/$REALM/users/${USER_ID}/reset-password -H "Authorization: Bearer ${ADMIN_ACCESS_TOKEN}" -H "Content-Type: application/json" -d "${CREDENTIALS_JSON}" + curl -X PUT -k $KEYCLOAK_BASE_URL/admin/realms/$REALM/users/${USER_ID}/reset-password -H "Authorization: Bearer ${ADMIN_ACCESS_TOKEN}" -H "Content-Type: application/json" -d "${CREDENTIALS_JSON}" done - \ No newline at end of file + diff --git a/tests/performance/load-tests.sh b/tests/performance/load-tests.sh index 2cf0f13313..1ffc751479 100755 --- a/tests/performance/load-tests.sh +++ b/tests/performance/load-tests.sh @@ -129,6 +129,7 @@ if [ ! -z $CRED_FILE ]; then fi # wait for ftp-server to be running +echo "wait for ftp server to be running" if [[ $clean_pvc == true ]]; then while [ true ] do @@ -141,6 +142,7 @@ if [[ $clean_pvc == true ]]; then fi # set common variables to template.yaml +echo "set common variables to template.yaml" cp pod.yaml template.yaml parsed_url=$(echo $URL | sed 's/\//\\\//g') parsed_image=$(echo $TEST_IMAGE | sed 's/\//\\\//g') @@ -223,7 +225,7 @@ echo "-- Gathering logs." echo "Syncing files from PVC to local folder." mkdir $FOLDER/$TIMESTAMP cd $FOLDER/$TIMESTAMP -oc rsync --no-perms --include "*.tar" ftp-server:/home/vsftpd/user/ $FOLDER/$TIMESTAMP +oc rsync --no-perms ftp-server:/home/vsftpd/user/ $FOLDER/$TIMESTAMP echo "Tar files rsynced, untarring..." for filename in *.tar; do tar xf $filename; @@ -231,20 +233,6 @@ done rm *.tar cd .. -# gather logs from pods -echo "Gathering logs from pods." -for p in $(oc get pods -l group=load-tests -o name) -do - IFS='-' - read -a strarr <<< "$p" - unset IFS - number=${strarr[2]} - path_to_logs="$FOLDER/$TIMESTAMP" - file="$path_to_logs/pod-$number-console-logs.txt" - touch $file - oc logs $p > $file -done - # ----------- CLEANING ENVIRONMENT ----------- # echo "-- Cleaning environment." diff --git a/tests/performance/sum_up_failures.sh b/tests/performance/sum_up_failures.sh new file mode 100755 index 0000000000..5a0edec30c --- /dev/null +++ b/tests/performance/sum_up_failures.sh @@ -0,0 +1,169 @@ +#!/bin/bash +# +# Copyright (c) 2020 Red Hat, Inc. +# This program and the accompanying materials are made +# available under the terms of the Eclipse Public License 2.0 +# which is available at https://www.eclipse.org/legal/epl-2.0/ +# +# SPDX-License-Identifier: EPL-2.0 +# +# See: https://sipb.mit.edu/doc/safe-shell/ + +# This script goes through results of load tests and sums up well known issues. Also reports non recognized failures for manual investigation. +# To run this script, set: +# TIMESTAMP to the name of a folder, where load tests results are stored. +# filedFolderNames to the array of a users, where test failed. You can easily copy that from the load test sum up. + +TIMESTAMP= +failedFolderNames=( ) + +staleErrors=0 +explorerErrs=0 +nosuchwindowErr=0 +stopErr=0 +removeErr=0 +loadErr=0 +tokenErr=0 +startErr=0 +ETIMEDOUTerr=0 + +sum=0 +staleElementUser="" +explorerUser="" +nosuchwindUser="" +stopErrUser="" +removeUser="" +loadErrUser="" +tokenErrUser="" +startErrUser="" +ETIMEDOUTUser="" +notclasified=0 + +#aditional info +usr409="" +usr503="" +usr504="" + +for i in "${failedFolderNames[@]}" +do + #################### Additional info gathering ############################################# + er409=$(cat report/$TIMESTAMP/$i/console-log.txt | grep '409') + if [[ ! -z $er409 ]]; then + usr409="$usr409 $i" + fi + er503=$(cat report/$TIMESTAMP/$i/console-log.txt | grep '503') + if [[ ! -z $er503 ]]; then + usr503="$usr503 $i" + fi + er504=$(cat report/$TIMESTAMP/$i/console-log.txt | grep '504') + if [[ ! -z $er504 ]]; then + usr504="$usr504 $i" + fi + ############################################################################################ + + sum=$((sum+1)) + staleerror=$(cat report/$TIMESTAMP/$i/console-log.txt | grep 'StaleElementReferenceError') + if [[ ! -z $staleerror ]]; then + staleErrors=$((staleErrors+1)) + staleElementUser="$staleElementUser $i" + continue; + fi + + explorer=$(cat report/$TIMESTAMP/$i/console-log.txt | grep "TimeoutError: Waiting for element to be located.*Explorer") + if [[ ! -z $explorer ]]; then + explorerErrs=$((explorerErrs+1)) + explorerUser="$explorerUser $i" + continue; + fi + + nosuchwindow=$(cat report/$TIMESTAMP/$i/console-log.txt | grep 'NoSuchWindowError: no such window') + if [[ ! -z $nosuchwindow ]]; then + nosuchwindowErr=$((nosuchwindowErr+1)) + nosuchwindUser="$nosuchwindUser $i" + continue; + fi + + loading=$(cat report/$TIMESTAMP/$i/console-log.txt | grep 'cannot determine loading status') + if [[ ! -z $loading ]]; then + loadErr=$((loadErr+1)) + loadErrUser="$loadErrUser $i" + continue; + fi + + starterror=$(cat report/$TIMESTAMP/$i/console-log.txt | grep ') Wait loading workspace and get time') + if [[ ! -z $starterror ]]; then + startErr=$((startErr+1)) + startErrUser="$startErrUser $i" + continue; + fi + + stoperror=$(cat report/$TIMESTAMP/$i/console-log.txt | grep 'Stopping workspace failed') + if [[ ! -z $stoperror ]]; then + stopErr=$((stopErr+1)) + stopErrUser="$stopErrUser $i" + continue; + fi + + removingerr=$(cat report/$TIMESTAMP/$i/console-log.txt | grep 'Removing of workspace failed') + if [[ ! -z $removingerr ]]; then + removeErr=$((removeErr+1)) + removeUser="$removeUser $i" + continue; + fi + + token=$(cat report/$TIMESTAMP/$i/console-log.txt | grep 'Can not get bearer token') + if [[ ! -z $token ]]; then + tokenErr=$((tokenErr+1)) + tokenErrUser="$tokenErrUser $i" + continue; + fi + + etimed=$(cat report/$TIMESTAMP/$i/console-log.txt | grep 'ETIMEDOUT') + if [[ ! -z $etimed ]]; then + ETIMEDOUTerr=$((ETIMEDOUTerr+1)) + ETIMEDOUTUser="$ETIMEDOUTUser $i" + continue; + fi + + + echo " ========================================================= vv $i vv ================================================================================" + cat report/$TIMESTAMP/$i/console-log.txt | grep ') "after all"' + notclasified=$((notclasified+1)) + +done + +echo "StaleElementReferenceError seen: $staleErrors times" +echo $staleElementUser +echo +echo "Waiting for IDE - Explorer seen: $explorerErrs times" +echo $explorerUser +echo +echo "NoSuchWindowError: no such window seen: $nosuchwindowErr times" +echo $nosuchwindUser +echo +echo "Failed to start workspace seen: $startErr times" +echo $startErrUser +echo +echo "Can not get bearer token seen: $tokenErr times" +echo $tokenErrUser +echo +echo "Stopping workspace failed seen: $stopErr times" +echo $stopErrUser +echo +echo "Removing of workspace failed seen: $removeErr times" +echo $removeUser +echo +echo "cannot determine loading status seen: $loadErr times" +echo $loadErrUser +echo +echo "afer suited - ETIMEDOUT seen: $ETIMEDOUTerr times" +echo $ETIMEDOUTUser +echo +echo "Remaining for manual verifictaion $notclasified/$sum" +echo +echo "+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++" +echo "info about error codes found:" +echo "409: $usr409" +echo "503: $usr503" +echo "504: $usr504" +echo "Note: error 409 happens when you try to remove workspace which is not stopped"