seg_yinzy
/
che-server
mirror of https://github.com/eclipse-che/che-server.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Print selenium test statistics in log during execution (#8883) 

Signed-off-by: Dmytro Nochevnov <dnochevnov@codenvy.com>
6.19.x
Dmytro Nochevnov 2018-02-23 10:40:56 +00:00 committed by GitHub
parent 3678c5fd56
commit f4b7ae5889
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 169 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

63
selenium/che-selenium-core/src/main/java/org/eclipse/che/selenium/core/inject/SeleniumTestHandler.java
View File

@ -127,6 +127,7 @@ public abstract class SeleniumTestHandler
@Inject private TestWorkspaceProvider testWorkspaceProvider;
@Inject private TestGitHubServiceClient gitHubClientService;
@Inject private TestWorkspaceLogsReader testWorkspaceLogsReader;
@Inject private SeleniumTestStatistics seleniumTestStatistics;
private final Injector injector;
private final Map<Long, Object> runningTests = new ConcurrentHashMap<>();
@ -150,25 +151,43 @@ public abstract class SeleniumTestHandler
}
@Override
public void onTestStart(ITestResult result) {}
public void onTestStart(ITestResult result) {
// count several invocations of method (e.g. in case of data provider is used)
String invocationLabel = "";
if (result.getMethod().getCurrentInvocationCount() > 0) {
invocationLabel = format(" (run %d)", result.getMethod().getCurrentInvocationCount() + 1);
}
LOG.info(
"Starting test #{} {}.{}{}. {}",
seleniumTestStatistics.hitStart(),
result.getTestClass().getRealClass().getSimpleName(),
result.getMethod().getMethodName(),
invocationLabel,
seleniumTestStatistics.toString());
}
@Override
public void onTestSuccess(ITestResult result) {
seleniumTestStatistics.hitPass();
onTestFinish(result);
}
@Override
public void onTestFailure(ITestResult result) {
seleniumTestStatistics.hitFail();
onTestFinish(result);
}
@Override
public void onTestSkipped(ITestResult result) {
seleniumTestStatistics.hitSkip();
onTestFinish(result);
}
@Override
public void onTestFailedButWithinSuccessPercentage(ITestResult result) {
seleniumTestStatistics.hitFail();
onTestFinish(result);
}
@ -181,6 +200,8 @@ public abstract class SeleniumTestHandler
@Override
public void onStart(ISuite suite) {
suite.setParentInjector(injector);
LOG.info(
"Starting suite '{}' with {} test methods.", suite.getName(), suite.getAllMethods().size());
}
/** Check if webdriver session can be created without errors. */
@ -257,19 +278,27 @@ public abstract class SeleniumTestHandler
/** Is invoked when test or configuration is finished. */
private void onTestFinish(ITestResult result) {
if (result.getStatus() == ITestResult.FAILURE || result.getStatus() == ITestResult.SKIP) {
String invocationLabel = "";
if (result.getMethod().getCurrentInvocationCount() > 1) {
invocationLabel = format(" (run %d)", result.getMethod().getCurrentInvocationCount());
}
if (result.getThrowable() != null) {
LOG.error(
"Test {} method {} failed because {}",
result.getTestClass().getName(),
"Test {}.{}{} failed. Error: {}",
result.getTestClass().getRealClass().getSimpleName(),
result.getMethod().getMethodName(),
invocationLabel,
result.getThrowable().getLocalizedMessage());
LOG.debug(result.getThrowable().getLocalizedMessage(), result.getThrowable());
} else {
LOG.error(
"Test {} method {} failed ",
result.getTestClass().getName(),
result.getMethod().getMethodName());
"Test {}.{}{} failed without exception.",
result.getTestClass().getRealClass().getSimpleName(),
result.getMethod().getMethodName(),
invocationLabel);
}
captureScreenshot(result);
captureHtmlSource(result);
captureTestWorkspaceLogs(result);
@ -286,7 +315,7 @@ public abstract class SeleniumTestHandler
obj = field.get(testInstance);
} catch (IllegalAccessException e) {
LOG.error(
"Field {} is unaccessable in {}.", field.getName(), testInstance.getClass().getName());
"Field {} is inaccessible in {}.", field.getName(), testInstance.getClass().getName());
continue;
}
@ -294,7 +323,7 @@ public abstract class SeleniumTestHandler
continue;
}
Path pathToStoreWorkspaceLogs = Paths.get(workspaceLogsDir, getTestDefinition(result));
Path pathToStoreWorkspaceLogs = Paths.get(workspaceLogsDir, getTestReference(result));
testWorkspaceLogsReader.read((TestWorkspace) obj, pathToStoreWorkspaceLogs);
}
}
@ -317,7 +346,7 @@ public abstract class SeleniumTestHandler
obj = field.get(testInstance);
} catch (IllegalAccessException e) {
LOG.error(
"Field {} is unaccessable in {}.", field.getName(), testInstance.getClass().getName());
"Field {} is inaccessible in {}.", field.getName(), testInstance.getClass().getName());
continue;
}
@ -380,7 +409,7 @@ public abstract class SeleniumTestHandler
obj = field.get(testInstance);
} catch (IllegalAccessException e) {
LOG.error(
"Field {} is unaccessable in {}.", field.getName(), testInstance.getClass().getName());
"Field {} is inaccessible in {}.", field.getName(), testInstance.getClass().getName());
continue;
}
@ -408,19 +437,19 @@ public abstract class SeleniumTestHandler
}
private void captureScreenshotFromWindow(ITestResult result, SeleniumWebDriver webDriver) {
String testDefinition = getTestDefinition(result);
String filename = NameGenerator.generate(testDefinition + "_", 8) + ".png";
String testReference = getTestReference(result);
String filename = NameGenerator.generate(testReference + "_", 8) + ".png";
try {
byte[] data = webDriver.getScreenshotAs(OutputType.BYTES);
Path screenshot = Paths.get(screenshotsDir, filename);
Files.createDirectories(screenshot.getParent());
Files.copy(new ByteArrayInputStream(data), screenshot);
} catch (WebDriverException | IOException e) {
LOG.error(format("Can't capture screenshot for test %s", testDefinition), e);
LOG.error(format("Can't capture screenshot for test %s", testReference), e);
}
}
private String getTestDefinition(ITestResult result) {
private String getTestReference(ITestResult result) {
return result.getTestClass().getName() + "." + result.getMethod().getMethodName();
}
@ -436,8 +465,8 @@ public abstract class SeleniumTestHandler
}
private void dumpHtmlCodeFromTheCurrentPage(ITestResult result, SeleniumWebDriver webDriver) {
String testName = getTestDefinition(result);
String filename = NameGenerator.generate(testName + "_", 8) + ".html";
String testReference = getTestReference(result);
String filename = NameGenerator.generate(testReference + "_", 8) + ".html";
try {
String pageSource = webDriver.getPageSource();
Path dumpDirectory = Paths.get(htmldumpsDir, filename);
@ -445,7 +474,7 @@ public abstract class SeleniumTestHandler
Files.write(
dumpDirectory, pageSource.getBytes(Charset.forName("UTF-8")), StandardOpenOption.CREATE);
} catch (WebDriverException | IOException e) {
LOG.error(format("Can't dump of html source for test %s", testName), e);
LOG.error(format("Can't dump of html source for test %s", testReference), e);
}
}

48
selenium/che-selenium-core/src/main/java/org/eclipse/che/selenium/core/inject/SeleniumTestStatistics.java Normal file
View File

@ -0,0 +1,48 @@
/*
* Copyright (c) 2012-2018 Red Hat, Inc.
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at
* http://www.eclipse.org/legal/epl-v10.html
*
* Contributors:
* Red Hat, Inc. - initial API and implementation
*/
package org.eclipse.che.selenium.core.inject;
import static java.lang.String.format;
import java.util.concurrent.atomic.AtomicInteger;
/** @author Dmytro Nochevnov */
public class SeleniumTestStatistics {
private static final String statisticsTemplate = "Passed: %d, failed: %d, skipped: %d.";
private final AtomicInteger runTests = new AtomicInteger();
private final AtomicInteger failedTests = new AtomicInteger();
private final AtomicInteger passedTests = new AtomicInteger();
private final AtomicInteger skippedTests = new AtomicInteger();
public int hitStart() {
return runTests.incrementAndGet();
}
public int hitPass() {
return passedTests.incrementAndGet();
}
public int hitFail() {
return failedTests.incrementAndGet();
}
public int hitSkip() {
return skippedTests.incrementAndGet();
}
@Override
public String toString() {
synchronized (this) {
return format(statisticsTemplate, passedTests.get(), failedTests.get(), skippedTests.get());
}
}
}

72
selenium/che-selenium-core/src/test/java/org/eclipse/che/selenium/core/inject/SeleniumTestStatisticsTest.java Normal file
View File

@ -0,0 +1,72 @@
/*
* Copyright (c) 2012-2018 Red Hat, Inc.
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at
* http://www.eclipse.org/legal/epl-v10.html
*
* Contributors:
* Red Hat, Inc. - initial API and implementation
*/
package org.eclipse.che.selenium.core.inject;
import static java.util.stream.IntStream.range;
import static org.testng.Assert.assertEquals;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
/** @author Dmytro Nochevnov */
public class SeleniumTestStatisticsTest {
private static final Logger LOG = LoggerFactory.getLogger(SeleniumTestStatisticsTest.class);
@Test(dataProvider = "testData")
@SuppressWarnings("FutureReturnValueIgnored")
public void testSimultaneousUsage(
int hitStart,
int hitPass,
int hitFail,
int hitSkip,
String initialToString,
String finalToString)
throws InterruptedException {
// given
SeleniumTestStatistics seleniumTestStatistics = new SeleniumTestStatistics();
// when
String actualString = seleniumTestStatistics.toString();
// then
assertEquals(actualString, initialToString);
// when
ExecutorService executor = Executors.newFixedThreadPool(10);
range(0, hitStart).forEach(i -> executor.submit(seleniumTestStatistics::hitStart));
range(0, hitPass).forEach(i -> executor.submit(seleniumTestStatistics::hitPass));
range(0, hitFail).forEach(i -> executor.submit(seleniumTestStatistics::hitFail));
range(0, hitSkip).forEach(i -> executor.submit(seleniumTestStatistics::hitSkip));
executor.awaitTermination(20, TimeUnit.SECONDS);
// then
assertEquals(seleniumTestStatistics.toString(), finalToString);
assertEquals(seleniumTestStatistics.hitStart(), hitStart + 1);
assertEquals(seleniumTestStatistics.hitPass(), hitPass + 1);
assertEquals(seleniumTestStatistics.hitFail(), hitFail + 1);
assertEquals(seleniumTestStatistics.hitSkip(), hitSkip + 1);
}
@DataProvider
public Object[][] testData() {
return new Object[][] {
{0, 0, 0, 0, "Passed: 0, failed: 0, skipped: 0.", "Passed: 0, failed: 0, skipped: 0."},
{31, 21, 5, 4, "Passed: 0, failed: 0, skipped: 0.", "Passed: 21, failed: 5, skipped: 4."}
};
}
}

4
selenium/che-selenium-core/src/test/java/org/eclipse/che/selenium/core/workspace/TestWorkspaceLogsReaderTest.java
View File

@ -37,6 +37,7 @@ import org.mockito.MockitoAnnotations;
import org.mockito.Spy;
import org.mockito.testng.MockitoTestNGListener;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Listeners;
import org.testng.annotations.Test;
@ -57,7 +58,8 @@ public class TestWorkspaceLogsReaderTest {
private static final String TEST_READ_SECOND_LOG_COMMAND = "echo 'read-log-2'";
private static final String UNKNOWN_COMMAND = "command-435f4q6we3as5va5s";
private static final String TEST_WORKSPACE_ID = "workspace-id";
private static final Path PATH_TO_STORE_LOGS = Paths.get("path-to-store-logs");
private static final Path PATH_TO_STORE_LOGS =
Paths.get(TestWorkspaceLogsReaderTest.class.getResource("").getPath());
private static final WorkspaceStatus WRONG_WORKSPACE_STATUS = WorkspaceStatus.STOPPED;
private static final NullPointerException EXCEPTION_TO_BE_THROWN = new NullPointerException();