diff --git a/selenium/che-selenium-core/src/main/java/org/eclipse/che/selenium/core/inject/SeleniumTestHandler.java b/selenium/che-selenium-core/src/main/java/org/eclipse/che/selenium/core/inject/SeleniumTestHandler.java index 71bd91a5d7..1f0f58b03c 100644 --- a/selenium/che-selenium-core/src/main/java/org/eclipse/che/selenium/core/inject/SeleniumTestHandler.java +++ b/selenium/che-selenium-core/src/main/java/org/eclipse/che/selenium/core/inject/SeleniumTestHandler.java @@ -127,6 +127,7 @@ public abstract class SeleniumTestHandler @Inject private TestWorkspaceProvider testWorkspaceProvider; @Inject private TestGitHubServiceClient gitHubClientService; @Inject private TestWorkspaceLogsReader testWorkspaceLogsReader; + @Inject private SeleniumTestStatistics seleniumTestStatistics; private final Injector injector; private final Map runningTests = new ConcurrentHashMap<>(); @@ -150,25 +151,43 @@ public abstract class SeleniumTestHandler } @Override - public void onTestStart(ITestResult result) {} + public void onTestStart(ITestResult result) { + // count several invocations of method (e.g. in case of data provider is used) + String invocationLabel = ""; + if (result.getMethod().getCurrentInvocationCount() > 0) { + invocationLabel = format(" (run %d)", result.getMethod().getCurrentInvocationCount() + 1); + } + + LOG.info( + "Starting test #{} {}.{}{}. {}", + seleniumTestStatistics.hitStart(), + result.getTestClass().getRealClass().getSimpleName(), + result.getMethod().getMethodName(), + invocationLabel, + seleniumTestStatistics.toString()); + } @Override public void onTestSuccess(ITestResult result) { + seleniumTestStatistics.hitPass(); onTestFinish(result); } @Override public void onTestFailure(ITestResult result) { + seleniumTestStatistics.hitFail(); onTestFinish(result); } @Override public void onTestSkipped(ITestResult result) { + seleniumTestStatistics.hitSkip(); onTestFinish(result); } @Override public void onTestFailedButWithinSuccessPercentage(ITestResult result) { + seleniumTestStatistics.hitFail(); onTestFinish(result); } @@ -181,6 +200,8 @@ public abstract class SeleniumTestHandler @Override public void onStart(ISuite suite) { suite.setParentInjector(injector); + LOG.info( + "Starting suite '{}' with {} test methods.", suite.getName(), suite.getAllMethods().size()); } /** Check if webdriver session can be created without errors. */ @@ -257,19 +278,27 @@ public abstract class SeleniumTestHandler /** Is invoked when test or configuration is finished. */ private void onTestFinish(ITestResult result) { if (result.getStatus() == ITestResult.FAILURE || result.getStatus() == ITestResult.SKIP) { + String invocationLabel = ""; + if (result.getMethod().getCurrentInvocationCount() > 1) { + invocationLabel = format(" (run %d)", result.getMethod().getCurrentInvocationCount()); + } + if (result.getThrowable() != null) { LOG.error( - "Test {} method {} failed because {}", - result.getTestClass().getName(), + "Test {}.{}{} failed. Error: {}", + result.getTestClass().getRealClass().getSimpleName(), result.getMethod().getMethodName(), + invocationLabel, result.getThrowable().getLocalizedMessage()); LOG.debug(result.getThrowable().getLocalizedMessage(), result.getThrowable()); } else { LOG.error( - "Test {} method {} failed ", - result.getTestClass().getName(), - result.getMethod().getMethodName()); + "Test {}.{}{} failed without exception.", + result.getTestClass().getRealClass().getSimpleName(), + result.getMethod().getMethodName(), + invocationLabel); } + captureScreenshot(result); captureHtmlSource(result); captureTestWorkspaceLogs(result); @@ -286,7 +315,7 @@ public abstract class SeleniumTestHandler obj = field.get(testInstance); } catch (IllegalAccessException e) { LOG.error( - "Field {} is unaccessable in {}.", field.getName(), testInstance.getClass().getName()); + "Field {} is inaccessible in {}.", field.getName(), testInstance.getClass().getName()); continue; } @@ -294,7 +323,7 @@ public abstract class SeleniumTestHandler continue; } - Path pathToStoreWorkspaceLogs = Paths.get(workspaceLogsDir, getTestDefinition(result)); + Path pathToStoreWorkspaceLogs = Paths.get(workspaceLogsDir, getTestReference(result)); testWorkspaceLogsReader.read((TestWorkspace) obj, pathToStoreWorkspaceLogs); } } @@ -317,7 +346,7 @@ public abstract class SeleniumTestHandler obj = field.get(testInstance); } catch (IllegalAccessException e) { LOG.error( - "Field {} is unaccessable in {}.", field.getName(), testInstance.getClass().getName()); + "Field {} is inaccessible in {}.", field.getName(), testInstance.getClass().getName()); continue; } @@ -380,7 +409,7 @@ public abstract class SeleniumTestHandler obj = field.get(testInstance); } catch (IllegalAccessException e) { LOG.error( - "Field {} is unaccessable in {}.", field.getName(), testInstance.getClass().getName()); + "Field {} is inaccessible in {}.", field.getName(), testInstance.getClass().getName()); continue; } @@ -408,19 +437,19 @@ public abstract class SeleniumTestHandler } private void captureScreenshotFromWindow(ITestResult result, SeleniumWebDriver webDriver) { - String testDefinition = getTestDefinition(result); - String filename = NameGenerator.generate(testDefinition + "_", 8) + ".png"; + String testReference = getTestReference(result); + String filename = NameGenerator.generate(testReference + "_", 8) + ".png"; try { byte[] data = webDriver.getScreenshotAs(OutputType.BYTES); Path screenshot = Paths.get(screenshotsDir, filename); Files.createDirectories(screenshot.getParent()); Files.copy(new ByteArrayInputStream(data), screenshot); } catch (WebDriverException | IOException e) { - LOG.error(format("Can't capture screenshot for test %s", testDefinition), e); + LOG.error(format("Can't capture screenshot for test %s", testReference), e); } } - private String getTestDefinition(ITestResult result) { + private String getTestReference(ITestResult result) { return result.getTestClass().getName() + "." + result.getMethod().getMethodName(); } @@ -436,8 +465,8 @@ public abstract class SeleniumTestHandler } private void dumpHtmlCodeFromTheCurrentPage(ITestResult result, SeleniumWebDriver webDriver) { - String testName = getTestDefinition(result); - String filename = NameGenerator.generate(testName + "_", 8) + ".html"; + String testReference = getTestReference(result); + String filename = NameGenerator.generate(testReference + "_", 8) + ".html"; try { String pageSource = webDriver.getPageSource(); Path dumpDirectory = Paths.get(htmldumpsDir, filename); @@ -445,7 +474,7 @@ public abstract class SeleniumTestHandler Files.write( dumpDirectory, pageSource.getBytes(Charset.forName("UTF-8")), StandardOpenOption.CREATE); } catch (WebDriverException | IOException e) { - LOG.error(format("Can't dump of html source for test %s", testName), e); + LOG.error(format("Can't dump of html source for test %s", testReference), e); } } diff --git a/selenium/che-selenium-core/src/main/java/org/eclipse/che/selenium/core/inject/SeleniumTestStatistics.java b/selenium/che-selenium-core/src/main/java/org/eclipse/che/selenium/core/inject/SeleniumTestStatistics.java new file mode 100644 index 0000000000..d3debfa878 --- /dev/null +++ b/selenium/che-selenium-core/src/main/java/org/eclipse/che/selenium/core/inject/SeleniumTestStatistics.java @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2012-2018 Red Hat, Inc. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Red Hat, Inc. - initial API and implementation + */ +package org.eclipse.che.selenium.core.inject; + +import static java.lang.String.format; + +import java.util.concurrent.atomic.AtomicInteger; + +/** @author Dmytro Nochevnov */ +public class SeleniumTestStatistics { + private static final String statisticsTemplate = "Passed: %d, failed: %d, skipped: %d."; + + private final AtomicInteger runTests = new AtomicInteger(); + private final AtomicInteger failedTests = new AtomicInteger(); + private final AtomicInteger passedTests = new AtomicInteger(); + private final AtomicInteger skippedTests = new AtomicInteger(); + + public int hitStart() { + return runTests.incrementAndGet(); + } + + public int hitPass() { + return passedTests.incrementAndGet(); + } + + public int hitFail() { + return failedTests.incrementAndGet(); + } + + public int hitSkip() { + return skippedTests.incrementAndGet(); + } + + @Override + public String toString() { + synchronized (this) { + return format(statisticsTemplate, passedTests.get(), failedTests.get(), skippedTests.get()); + } + } +} diff --git a/selenium/che-selenium-core/src/test/java/org/eclipse/che/selenium/core/inject/SeleniumTestStatisticsTest.java b/selenium/che-selenium-core/src/test/java/org/eclipse/che/selenium/core/inject/SeleniumTestStatisticsTest.java new file mode 100644 index 0000000000..cc8fbaec85 --- /dev/null +++ b/selenium/che-selenium-core/src/test/java/org/eclipse/che/selenium/core/inject/SeleniumTestStatisticsTest.java @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2012-2018 Red Hat, Inc. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Red Hat, Inc. - initial API and implementation + */ +package org.eclipse.che.selenium.core.inject; + +import static java.util.stream.IntStream.range; +import static org.testng.Assert.assertEquals; + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +/** @author Dmytro Nochevnov */ +public class SeleniumTestStatisticsTest { + private static final Logger LOG = LoggerFactory.getLogger(SeleniumTestStatisticsTest.class); + + @Test(dataProvider = "testData") + @SuppressWarnings("FutureReturnValueIgnored") + public void testSimultaneousUsage( + int hitStart, + int hitPass, + int hitFail, + int hitSkip, + String initialToString, + String finalToString) + throws InterruptedException { + // given + SeleniumTestStatistics seleniumTestStatistics = new SeleniumTestStatistics(); + + // when + String actualString = seleniumTestStatistics.toString(); + + // then + assertEquals(actualString, initialToString); + + // when + ExecutorService executor = Executors.newFixedThreadPool(10); + + range(0, hitStart).forEach(i -> executor.submit(seleniumTestStatistics::hitStart)); + range(0, hitPass).forEach(i -> executor.submit(seleniumTestStatistics::hitPass)); + range(0, hitFail).forEach(i -> executor.submit(seleniumTestStatistics::hitFail)); + range(0, hitSkip).forEach(i -> executor.submit(seleniumTestStatistics::hitSkip)); + + executor.awaitTermination(20, TimeUnit.SECONDS); + + // then + assertEquals(seleniumTestStatistics.toString(), finalToString); + assertEquals(seleniumTestStatistics.hitStart(), hitStart + 1); + assertEquals(seleniumTestStatistics.hitPass(), hitPass + 1); + assertEquals(seleniumTestStatistics.hitFail(), hitFail + 1); + assertEquals(seleniumTestStatistics.hitSkip(), hitSkip + 1); + } + + @DataProvider + public Object[][] testData() { + return new Object[][] { + {0, 0, 0, 0, "Passed: 0, failed: 0, skipped: 0.", "Passed: 0, failed: 0, skipped: 0."}, + {31, 21, 5, 4, "Passed: 0, failed: 0, skipped: 0.", "Passed: 21, failed: 5, skipped: 4."} + }; + } +} diff --git a/selenium/che-selenium-core/src/test/java/org/eclipse/che/selenium/core/workspace/TestWorkspaceLogsReaderTest.java b/selenium/che-selenium-core/src/test/java/org/eclipse/che/selenium/core/workspace/TestWorkspaceLogsReaderTest.java index 53fd3e555a..7d5b611d02 100644 --- a/selenium/che-selenium-core/src/test/java/org/eclipse/che/selenium/core/workspace/TestWorkspaceLogsReaderTest.java +++ b/selenium/che-selenium-core/src/test/java/org/eclipse/che/selenium/core/workspace/TestWorkspaceLogsReaderTest.java @@ -37,6 +37,7 @@ import org.mockito.MockitoAnnotations; import org.mockito.Spy; import org.mockito.testng.MockitoTestNGListener; import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Listeners; import org.testng.annotations.Test; @@ -57,7 +58,8 @@ public class TestWorkspaceLogsReaderTest { private static final String TEST_READ_SECOND_LOG_COMMAND = "echo 'read-log-2'"; private static final String UNKNOWN_COMMAND = "command-435f4q6we3as5va5s"; private static final String TEST_WORKSPACE_ID = "workspace-id"; - private static final Path PATH_TO_STORE_LOGS = Paths.get("path-to-store-logs"); + private static final Path PATH_TO_STORE_LOGS = + Paths.get(TestWorkspaceLogsReaderTest.class.getResource("").getPath()); private static final WorkspaceStatus WRONG_WORKSPACE_STATUS = WorkspaceStatus.STOPPED; private static final NullPointerException EXCEPTION_TO_BE_THROWN = new NullPointerException();