seg_yinzy
/
che-server
mirror of https://github.com/eclipse-che/che-server.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

CHE-4101 Rework docker infrastructure to use bootstrapper to start installers

6.19.x
Sergii Leshchenko 2017-06-23 15:23:41 +03:00
parent fbd1b02f25
commit fa48da7d74
14 changed files with 316 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
agents/che-core-api-agent/src/main/java/org/eclipse/che/api/agent/server/model/impl/AgentImpl.java
View File

@ -19,8 +19,6 @@ import java.util.List;
import java.util.Map;
import java.util.Objects;
import static java.util.stream.Collectors.toMap;
/**
* @author Anatoliy Bazko
*/
@ -50,9 +48,7 @@ public class AgentImpl implements Agent {
this.properties = properties;
this.script = script;
if (servers != null) {
this.servers = servers.entrySet()
.stream()
.collect(toMap(Map.Entry::getKey, Map.Entry::getValue));
this.servers = new HashMap<>(servers);
}
}

25
assembly/assembly-wsmaster-war/pom.xml
View File

@ -62,6 +62,12 @@
<groupId>javax.inject</groupId>
<artifactId>javax.inject</artifactId>
</dependency>
<dependency>
<groupId>org.eclipse.che</groupId>
<artifactId>bootstrapper</artifactId>
<type>tar.gz</type>
<classifier>linux_amd64</classifier>
</dependency>
<dependency>
<groupId>org.eclipse.che</groupId>
<artifactId>exec-agent</artifactId>
@ -312,6 +318,25 @@
<skip>true</skip>
</configuration>
</execution>
<execution>
<id>copy-bootstrapper</id>
<phase>process-resources</phase>
<goals>
<goal>copy</goal>
</goals>
<configuration>
<artifactItems>
<artifactItem>
<groupId>org.eclipse.che</groupId>
<artifactId>bootstrapper</artifactId>
<classifier>linux_amd64</classifier>
<type>tar.gz</type>
<destFileName>bootstrapper.tar.gz</destFileName>
</artifactItem>
</artifactItems>
<outputDirectory>${project.build.directory}/classes</outputDirectory>
</configuration>
</execution>
</executions>
</plugin>
<plugin>

17
assembly/assembly-wsmaster-war/src/main/webapp/WEB-INF/classes/codenvy/che.properties
View File

@ -103,6 +103,9 @@ che.workspace.hosts=NULL
# che-host is a hostname entry added to /etc/hosts of the workspace by the Che server.
che.workspace.che_server_endpoint=http://che-host:${SERVER_PORT}/wsmaster/api
# This is the webscoket base endpoint of the workspace master running within the core Che server.
che.workspace.che_server_websocket_endpoint_base=ws://che-host:${SERVER_PORT}/wsmaster
### AGENTS
# When the Che server launches a new workspace, Che pings a mini Che server running inside of the
# workspace runtime. We call this mini-Che an "agent". The Che server knows that the workspace
@ -259,6 +262,20 @@ che.docker.tcp_connection_read_timeout_ms=600000
# to determine swap size. To disable swap set to 0.
che.docker.swap=-1
### Che docker infrastructure parameters
# Time (in minutes) given for bootstrapping.
# If boostrapping is not finished in time it will be failed and workspace start will fail.
che.infra.docker.bootstrapper.timeout_min=10
# Time (in seconds) given for one installer to complete its installation.
# If installation is not finished in time it will be interrupted.
che.infra.docker.bootstrapper.installer_timeout_sec=180
# Time(in seconds) between servers availability checks.
# Once servers for one installer available - checks stopped.
che.infra.docker.bootstrapper.server_check_period_sec=3
### INTERNAL
# Remove locations where internal message bus events should be propagated to.
# For debugging - set to retrieve internal events from external clients.

4
core/che-core-api-model/src/main/java/org/eclipse/che/api/core/model/workspace/runtime/BootstrapperStatus.java
View File

@ -18,9 +18,9 @@ package org.eclipse.che.api.core.model.workspace.runtime;
public enum BootstrapperStatus {
/**
* Bootstrapper is ready to work, start installers, push events, etc.
* Bootstrapping is in progress.
*/
AVAILABLE,
STARTING,
/**
* Bootstrapping done, everything is started ok.

4
infrastructures/docker/pom.xml
View File

@ -38,6 +38,10 @@
<groupId>com.fasterxml.jackson.dataformat</groupId>
<artifactId>jackson-dataformat-yaml</artifactId>
</dependency>
<dependency>
<groupId>com.google.code.gson</groupId>
<artifactId>gson</artifactId>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>

193
infrastructures/docker/src/main/java/org/eclipse/che/workspace/infrastructure/docker/Bootstrapper.java Normal file
View File

@ -0,0 +1,193 @@
/*******************************************************************************
* Copyright (c) 2012-2017 Codenvy, S.A.
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at
* http://www.eclipse.org/legal/epl-v10.html
*
* Contributors:
* Codenvy, S.A. - initial API and implementation
*******************************************************************************/
package org.eclipse.che.workspace.infrastructure.docker;
import com.google.gson.Gson;
import com.google.inject.assistedinject.Assisted;
import org.eclipse.che.api.agent.server.model.impl.AgentImpl;
import org.eclipse.che.api.core.model.workspace.runtime.BootstrapperStatus;
import org.eclipse.che.api.core.model.workspace.runtime.RuntimeIdentity;
import org.eclipse.che.api.core.notification.EventService;
import org.eclipse.che.api.core.notification.EventSubscriber;
import org.eclipse.che.api.core.util.FileCleaner;
import org.eclipse.che.api.workspace.server.spi.InfrastructureException;
import org.eclipse.che.api.workspace.server.spi.InternalInfrastructureException;
import org.eclipse.che.api.workspace.shared.dto.RuntimeIdentityDto;
import org.eclipse.che.api.workspace.shared.dto.event.BootstrapperStatusEvent;
import org.eclipse.che.commons.lang.TarUtils;
import org.eclipse.che.workspace.infrastructure.docker.server.InstallerEndpoint;
import javax.inject.Inject;
import javax.inject.Named;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicInteger;
/**
* Bootstraps installers.
*
* @author Sergii Leshchenko
*/
public class Bootstrapper {
private static final Gson GSON = new Gson();
private static final AtomicInteger ENDPOINT_IDS = new AtomicInteger();
private static final String BOOTSTRAPPER_BASE_DIR = "/tmp/";
private static final String BOOTSTRAPPER_DIR = BOOTSTRAPPER_BASE_DIR + "bootstrapper/";
private static final String BOOTSTRAPPER_FILE = "bootstrapper";
private static final String CONFIG_FILE = "config.json";
private final String machineName;
private final RuntimeIdentity runtimeIdentity;
private final DockerMachine dockerMachine;
private final List<AgentImpl> agents;
private final int bootstrappingTimeoutMinutes;
private final int serverCheckPeriodSeconds;
private final int installerTimeoutSeconds;
private final String installerEndpoint;
private final EventService eventService;
private final EventSubscriber<BootstrapperStatusEvent> bootstrapperStatusListener;
private CompletableFuture<BootstrapperStatusEvent> finishEventFuture;
@Inject
public Bootstrapper(@Assisted String machineName,
@Assisted RuntimeIdentity runtimeIdentity,
@Assisted DockerMachine dockerMachine,
@Assisted List<AgentImpl> agents,
@Named("che.workspace.che_server_websocket_endpoint_base") String websocketBaseEndpoint,
@Named("che.infra.docker.bootstrapper.timeout_min") int bootstrappingTimeoutMinutes,
@Named("che.infra.docker.bootstrapper.installer_timeout_sec") int installerTimeoutSeconds,
@Named("che.infra.docker.bootstrapper.server_check_period_sec") int serverCheckPeriodSeconds,
EventService eventService) {
this.machineName = machineName;
this.runtimeIdentity = runtimeIdentity;
this.dockerMachine = dockerMachine;
this.agents = agents;
this.installerEndpoint = websocketBaseEndpoint + InstallerEndpoint.INSTALLER_WEBSOCKET_ENDPOINT_BASE;
this.bootstrappingTimeoutMinutes = bootstrappingTimeoutMinutes;
this.serverCheckPeriodSeconds = serverCheckPeriodSeconds;
this.installerTimeoutSeconds = installerTimeoutSeconds;
this.eventService = eventService;
this.bootstrapperStatusListener = event -> {
BootstrapperStatus status = event.getStatus();
//skip starting status event
if (status.equals(BootstrapperStatus.DONE) || status.equals(BootstrapperStatus.FAILED)) {
//check boostrapper belongs to current runtime and machine
RuntimeIdentityDto runtimeId = event.getRuntimeId();
if (event.getMachineName().equals(machineName)
&& runtimeIdentity.getEnvName().equals(runtimeId.getEnvName())
&& runtimeIdentity.getOwner().equals(runtimeId.getOwner())
&& runtimeIdentity.getWorkspaceId().equals(runtimeId.getWorkspaceId())) {
finishEventFuture.complete(event);
}
}
};
}
/**
* Bootstraps installers and wait while they finished.
*
* @throws InfrastructureException
* when bootstrapping timeout reached
* @throws InfrastructureException
* when bootstrapping failed
* @throws InfrastructureException
* when any other error occurs while bootstrapping
*/
public void bootstrap() throws InfrastructureException {
if (finishEventFuture != null) {
throw new IllegalStateException("Bootstrap method must be called only once.");
}
this.finishEventFuture = new CompletableFuture<>();
injectBootstrapper();
this.eventService.subscribe(bootstrapperStatusListener, BootstrapperStatusEvent.class);
try {
String endpoint = installerEndpoint + ENDPOINT_IDS.getAndIncrement();
dockerMachine.exec(BOOTSTRAPPER_DIR + BOOTSTRAPPER_FILE +
" -machine-name " + machineName +
" -runtime-id " + String.format("%s:%s:%s", runtimeIdentity.getWorkspaceId(),
runtimeIdentity.getEnvName(),
runtimeIdentity.getOwner()) +
" -push-endpoint " + endpoint +
" -push-logs-endpoint " + endpoint +
" -server-check-period " + serverCheckPeriodSeconds +
" -installer-timeout " + installerTimeoutSeconds +
" -file " + BOOTSTRAPPER_DIR + CONFIG_FILE,
null);
//waiting for DONE or FAILED bootstrapper status event
BootstrapperStatusEvent resultEvent = finishEventFuture.get(bootstrappingTimeoutMinutes, TimeUnit.MINUTES);
if (resultEvent.getStatus().equals(BootstrapperStatus.FAILED)) {
throw new InfrastructureException(resultEvent.getError());
}
} catch (ExecutionException e) {
throw new InfrastructureException(e.getCause().getMessage(), e);
} catch (TimeoutException e) {
throw new InfrastructureException("Bootstrapping of machine " + machineName + " reached timeout");
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new InfrastructureException("Bootstrapping of machine " + machineName + " was interrupted");
} finally {
this.eventService.unsubscribe(bootstrapperStatusListener, BootstrapperStatusEvent.class);
}
}
private void injectBootstrapper() throws InfrastructureException {
dockerMachine.putResource(BOOTSTRAPPER_BASE_DIR, Thread.currentThread()
.getContextClassLoader()
.getResourceAsStream("bootstrapper.tar.gz"));
// inject config file
File configFileArchive = null;
try {
configFileArchive = createArchive(CONFIG_FILE, GSON.toJson(agents));
dockerMachine.putResource(BOOTSTRAPPER_DIR, new FileInputStream(configFileArchive));
} catch (FileNotFoundException e) {
throw new InternalInfrastructureException(e.getMessage(), e);
} finally {
if (configFileArchive != null) {
FileCleaner.addFile(configFileArchive);
}
}
}
private File createArchive(String filename, String content) throws InfrastructureException {
Path bootstrapperConfTmp = null;
try {
bootstrapperConfTmp = Files.createTempDirectory(filename);
Path configFile = bootstrapperConfTmp.resolve(filename);
Files.copy(new ByteArrayInputStream(content.getBytes()), configFile);
Path bootstrapperConfArchive = Files.createTempFile(filename, ".tar.gz");
TarUtils.tarFiles(bootstrapperConfArchive.toFile(), configFile.toFile());
return bootstrapperConfArchive.toFile();
} catch (IOException e) {
throw new InternalInfrastructureException("Error occurred while injecting bootstrapping conf. "
+ e.getMessage(), e);
} finally {
if (bootstrapperConfTmp != null) {
FileCleaner.addFile(bootstrapperConfTmp.toFile());
}
}
}
}

28
infrastructures/docker/src/main/java/org/eclipse/che/workspace/infrastructure/docker/BootstrapperFactory.java Normal file
View File

@ -0,0 +1,28 @@
/*******************************************************************************
* Copyright (c) 2012-2017 Codenvy, S.A.
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at
* http://www.eclipse.org/legal/epl-v10.html
*
* Contributors:
* Codenvy, S.A. - initial API and implementation
*******************************************************************************/
package org.eclipse.che.workspace.infrastructure.docker;
import com.google.inject.assistedinject.Assisted;
import org.eclipse.che.api.agent.server.model.impl.AgentImpl;
import org.eclipse.che.api.core.model.workspace.runtime.RuntimeIdentity;
import java.util.List;
/**
* @author Sergii Leshchenko
*/
public interface BootstrapperFactory {
Bootstrapper create(@Assisted String machineName,
@Assisted RuntimeIdentity runtimeIdentity,
@Assisted DockerMachine dockerMachine,
@Assisted List<AgentImpl> agents);
}

4
infrastructures/docker/src/main/java/org/eclipse/che/workspace/infrastructure/docker/DockerInfraModule.java
View File

@ -108,5 +108,9 @@ public class DockerInfraModule extends AbstractModule {
install(new FactoryModuleBuilder()
.implement(DockerInternalRuntime.class, DockerInternalRuntime.class)
.build(DockerRuntimeFactory.class));
install(new FactoryModuleBuilder()
.implement(Bootstrapper.class, Bootstrapper.class)
.build(BootstrapperFactory.class));
}
}

39
infrastructures/docker/src/main/java/org/eclipse/che/workspace/infrastructure/docker/DockerInternalRuntime.java
View File

@ -13,7 +13,6 @@ package org.eclipse.che.workspace.infrastructure.docker;
import com.google.common.collect.ImmutableMap;
import com.google.inject.assistedinject.Assisted;
import org.eclipse.che.api.agent.shared.model.impl.AgentImpl;
import org.eclipse.che.api.core.NotFoundException;
import org.eclipse.che.api.core.model.machine.MachineSource;
import org.eclipse.che.api.core.model.workspace.runtime.Machine;
@ -33,7 +32,6 @@ import org.eclipse.che.api.workspace.server.spi.InternalRuntime;
import org.eclipse.che.api.workspace.shared.dto.event.MachineStatusEvent;
import org.eclipse.che.api.workspace.shared.dto.event.ServerStatusEvent;
import org.eclipse.che.dto.server.DtoFactory;
import org.eclipse.che.plugin.docker.client.MessageProcessor;
import org.eclipse.che.workspace.infrastructure.docker.exception.SourceNotFoundException;
import org.eclipse.che.workspace.infrastructure.docker.model.DockerBuildContext;
import org.eclipse.che.workspace.infrastructure.docker.model.DockerContainerConfig;
@ -85,6 +83,7 @@ public class DockerInternalRuntime extends InternalRuntime<DockerRuntimeContext>
private final DockerRegistryClient dockerRegistryClient;
private final RuntimeIdentity identity;
private final EventService eventService;
private final BootstrapperFactory bootstrapperFactory;
@Inject
public DockerInternalRuntime(@Assisted DockerRuntimeContext context,
@ -98,12 +97,14 @@ public class DockerInternalRuntime extends InternalRuntime<DockerRuntimeContext>
DockerMachineStarter serviceStarter,
SnapshotDao snapshotDao,
DockerRegistryClient dockerRegistryClient,
EventService eventService) {
EventService eventService,
BootstrapperFactory bootstrapperFactory) {
super(context, urlRewriter);
this.devMachineName = devMachineName;
this.dockerEnvironment = dockerEnvironment;
this.identity = identity;
this.eventService = eventService;
this.bootstrapperFactory = bootstrapperFactory;
this.properties = new HashMap<>();
this.startSynchronizer = new StartSynchronizer();
this.contextsStorage = contextsStorage;
@ -235,34 +236,24 @@ public class DockerInternalRuntime extends InternalRuntime<DockerRuntimeContext>
identity,
isDev);
}
try {
checkStartInterruption();
startSynchronizer.addMachine(name, dockerMachine);
InternalMachineConfig machineConfig = getContext().getMachineConfigs().get(name);
if (machineConfig == null) {
throw new InfrastructureException("Machine %s is not found in internal machines config of RuntimeContext");
}
bootstrapperFactory.create(name, identity, dockerMachine, machineConfig.getAgents())
.bootstrap();
checkServersReadiness(name, dockerMachine);
} catch (InfrastructureException e) {
destroyMachineQuietly(name, dockerMachine);
throw e;
}
startAgents(name, dockerMachine);
checkServersReadiness(name, dockerMachine);
}
// TODO rework to agent launchers
private void startAgents(String machineName, DockerMachine dockerMachine) throws InfrastructureException {
InternalMachineConfig machineConfig = getContext().getMachineConfigs().get(machineName);
if (machineConfig == null) {
throw new InfrastructureException("Machine %s is not found in internal machines config of RuntimeContext");
}
for (AgentImpl agent : machineConfig.getAgents()) {
Thread thread = new Thread(() -> {
try {
dockerMachine.exec(agent.getScript(), MessageProcessor.getDevNull());
} catch (InfrastructureException e) {
LOG.error(e.getLocalizedMessage(), e);
}
});
thread.setDaemon(true);
thread.start();
}
}
private void checkServersReadiness(String machineName, DockerMachine dockerMachine)

10
infrastructures/docker/src/main/java/org/eclipse/che/workspace/infrastructure/docker/DockerMachine.java
View File

@ -27,6 +27,7 @@ import org.eclipse.che.plugin.docker.client.json.ContainerInfo;
import org.eclipse.che.plugin.docker.client.params.CommitParams;
import org.eclipse.che.plugin.docker.client.params.CreateExecParams;
import org.eclipse.che.plugin.docker.client.params.PushParams;
import org.eclipse.che.plugin.docker.client.params.PutResourceParams;
import org.eclipse.che.plugin.docker.client.params.RemoveContainerParams;
import org.eclipse.che.plugin.docker.client.params.RemoveImageParams;
import org.eclipse.che.plugin.docker.client.params.StartExecParams;
@ -38,6 +39,7 @@ import org.slf4j.Logger;
import javax.inject.Inject;
import java.io.IOException;
import java.io.InputStream;
import java.util.Collections;
import java.util.Map;
@ -148,6 +150,14 @@ public class DockerMachine implements Machine {
server.setStatus(status);
}
public void putResource(String targetPath, InputStream sourceStream) throws InfrastructureException {
try {
docker.putResource(PutResourceParams.create(container, targetPath, sourceStream));
} catch (IOException e) {
throw new InternalInfrastructureException(e.getMessage(), e);
}
}
public void exec(String script, MessageProcessor<LogMessage> messageProcessor) throws InfrastructureException {
try {
Exec exec = docker.createExec(CreateExecParams.create(container,

4
infrastructures/docker/src/main/java/org/eclipse/che/workspace/infrastructure/docker/server/InstallerEndpoint.java
View File

@ -25,9 +25,11 @@ import javax.websocket.server.ServerEndpoint;
* @author Max Shaposhnik (mshaposhnik@codenvy.com)
*/
@ServerEndpoint(value = "/installer/websocket/{endpoint-id}", configurator = GuiceInjectorEndpointConfigurator.class)
@ServerEndpoint(value = InstallerEndpoint.INSTALLER_WEBSOCKET_ENDPOINT_BASE + "{endpoint-id}", configurator = GuiceInjectorEndpointConfigurator.class)
public class InstallerEndpoint extends BasicWebSocketEndpoint {
public static final String INSTALLER_WEBSOCKET_ENDPOINT_BASE = "/installer/websocket/";
@Inject
public InstallerEndpoint(WebSocketSessionRegistry registry,
MessagesReSender reSender,

12
pom.xml
View File

@ -106,6 +106,18 @@
<version>${che.version}</version>
<type>war</type>
</dependency>
<dependency>
<groupId>org.eclipse.che</groupId>
<artifactId>bootstrapper</artifactId>
<version>${che.version}</version>
</dependency>
<dependency>
<groupId>org.eclipse.che</groupId>
<artifactId>bootstrapper</artifactId>
<version>${che.version}</version>
<type>tar.gz</type>
<classifier>linux_amd64</classifier>
</dependency>
<dependency>
<groupId>org.eclipse.che</groupId>
<artifactId>exec-agent</artifactId>

8
wsmaster/che-core-api-workspace-shared/src/main/java/org/eclipse/che/api/workspace/shared/dto/event/BootstrapperStatusEvent.java
View File

@ -28,14 +28,6 @@ public interface BootstrapperStatusEvent {
BootstrapperStatusEvent withStatus(BootstrapperStatus status);
String getInstaller();
void setInstaller(String installer);
BootstrapperStatusEvent withInstaller(String installer);
String getMachineName();
void setMachineName(String machineName);

4
wsmaster/che-core-api-workspace/src/main/java/org/eclipse/che/api/workspace/server/spi/InternalMachineConfig.java
View File

@ -53,8 +53,8 @@ public class InternalMachineConfig {
this.servers.putAll(originalConfig.getServers());
this.attributes = new HashMap<>(originalConfig.getAttributes());
if(agentRegistry != null && agentSorter != null)
initAgents(originalConfig.getAgents());
if (agentRegistry != null && agentSorter != null)
initAgents(originalConfig.getAgents());
}
/**