CHE-5093: hard code server status check in docker infra (#5405)
Add hard coded server status check in Docker infrastructure to test usage of server statuses events in client load sequence. Add server URL into ServerStatueEvent to ease workflow of the client. Add more info about agent into the internal representation of agent. Use assisted injection to clean up docker infra code. Allow accessing Docker machine before agents start to prevent crashes of wsagent.6.19.x
Alexander Garagatyi
GitHub
parent
2fc5db467b
commit
0993fb8a2b
|
|
@ -29,6 +29,5 @@ public enum ServerStatus {
|
|||
/**
|
||||
* unknown
|
||||
*/
|
||||
UNKNOWN;
|
||||
|
||||
UNKNOWN
|
||||
}
|
||||
|
|
|
|||
|
|
@ -106,7 +106,7 @@ public class DockerInfraModule extends AbstractModule {
|
|||
bind(DockerRegistryDynamicAuthResolver.class).to(NoOpDockerRegistryDynamicAuthResolverImpl.class);
|
||||
|
||||
install(new FactoryModuleBuilder()
|
||||
.implement(DockerRuntimeContext.class, DockerRuntimeContext.class)
|
||||
.build(RuntimeFactory.class));
|
||||
.implement(DockerInternalRuntime.class, DockerInternalRuntime.class)
|
||||
.build(DockerRuntimeFactory.class));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -10,20 +10,28 @@
|
|||
*******************************************************************************/
|
||||
package org.eclipse.che.workspace.infrastructure.docker;
|
||||
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.inject.assistedinject.Assisted;
|
||||
|
||||
import org.eclipse.che.api.agent.shared.model.impl.AgentImpl;
|
||||
import org.eclipse.che.api.core.NotFoundException;
|
||||
import org.eclipse.che.api.core.model.machine.MachineSource;
|
||||
import org.eclipse.che.api.core.model.workspace.runtime.Machine;
|
||||
import org.eclipse.che.api.core.model.workspace.runtime.MachineStatus;
|
||||
import org.eclipse.che.api.core.model.workspace.runtime.RuntimeIdentity;
|
||||
import org.eclipse.che.api.core.model.workspace.runtime.ServerStatus;
|
||||
import org.eclipse.che.api.core.notification.EventService;
|
||||
import org.eclipse.che.api.workspace.server.DtoConverter;
|
||||
import org.eclipse.che.api.workspace.server.URLRewriter;
|
||||
import org.eclipse.che.api.workspace.server.model.impl.MachineImpl;
|
||||
import org.eclipse.che.api.workspace.server.model.impl.MachineSourceImpl;
|
||||
import org.eclipse.che.api.workspace.server.model.impl.ServerImpl;
|
||||
import org.eclipse.che.api.workspace.server.spi.InfrastructureException;
|
||||
import org.eclipse.che.api.workspace.server.spi.InternalInfrastructureException;
|
||||
import org.eclipse.che.api.workspace.server.spi.InternalMachineConfig;
|
||||
import org.eclipse.che.api.workspace.server.spi.InternalRuntime;
|
||||
import org.eclipse.che.api.workspace.shared.dto.event.MachineStatusEvent;
|
||||
import org.eclipse.che.api.workspace.shared.dto.event.ServerStatusEvent;
|
||||
import org.eclipse.che.dto.server.DtoFactory;
|
||||
import org.eclipse.che.plugin.docker.client.MessageProcessor;
|
||||
import org.eclipse.che.workspace.infrastructure.docker.exception.SourceNotFoundException;
|
||||
|
|
@ -35,6 +43,12 @@ import org.eclipse.che.workspace.infrastructure.docker.snapshot.SnapshotExceptio
|
|||
import org.eclipse.che.workspace.infrastructure.docker.snapshot.SnapshotImpl;
|
||||
import org.slf4j.Logger;
|
||||
|
||||
import javax.inject.Inject;
|
||||
import javax.ws.rs.core.UriBuilder;
|
||||
import java.io.IOException;
|
||||
import java.net.HttpURLConnection;
|
||||
import java.net.MalformedURLException;
|
||||
import java.net.URL;
|
||||
import java.util.ArrayDeque;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
|
|
@ -45,6 +59,7 @@ import java.util.Map;
|
|||
import java.util.Queue;
|
||||
|
||||
import static java.lang.String.format;
|
||||
import static java.util.stream.Collectors.toMap;
|
||||
import static org.slf4j.LoggerFactory.getLogger;
|
||||
|
||||
/**
|
||||
|
|
@ -54,6 +69,9 @@ public class DockerInternalRuntime extends InternalRuntime<DockerRuntimeContext>
|
|||
|
||||
private static final Logger LOG = getLogger(DockerInternalRuntime.class);
|
||||
|
||||
private static Map<String, String> livenessChecksPaths = ImmutableMap.of("wsagent", "/api/",
|
||||
"exec-agent", "/process");
|
||||
|
||||
private final StartSynchronizer startSynchronizer;
|
||||
private final Map<String, String> properties;
|
||||
private final Queue<String> startQueue;
|
||||
|
|
@ -61,23 +79,24 @@ public class DockerInternalRuntime extends InternalRuntime<DockerRuntimeContext>
|
|||
private final NetworkLifecycle dockerNetworkLifecycle;
|
||||
private final String devMachineName;
|
||||
private final DockerEnvironment dockerEnvironment;
|
||||
private final MachineStarter serviceStarter;
|
||||
private final DockerMachineStarter serviceStarter;
|
||||
private final SnapshotDao snapshotDao;
|
||||
private final DockerRegistryClient dockerRegistryClient;
|
||||
private final RuntimeIdentity identity;
|
||||
private final EventService eventService;
|
||||
|
||||
public DockerInternalRuntime(DockerRuntimeContext context,
|
||||
String devMachineName,
|
||||
@Inject
|
||||
public DockerInternalRuntime(@Assisted DockerRuntimeContext context,
|
||||
@Assisted String devMachineName,
|
||||
@Assisted List<String> orderedServices,
|
||||
@Assisted DockerEnvironment dockerEnvironment,
|
||||
@Assisted RuntimeIdentity identity,
|
||||
URLRewriter urlRewriter,
|
||||
List<String> orderedServices,
|
||||
ContextsStorage contextsStorage,
|
||||
DockerEnvironment dockerEnvironment,
|
||||
NetworkLifecycle dockerNetworkLifecycle,
|
||||
MachineStarter serviceStarter,
|
||||
DockerMachineStarter serviceStarter,
|
||||
SnapshotDao snapshotDao,
|
||||
DockerRegistryClient dockerRegistryClient,
|
||||
RuntimeIdentity identity,
|
||||
EventService eventService) {
|
||||
super(context, urlRewriter);
|
||||
this.devMachineName = devMachineName;
|
||||
|
|
@ -103,7 +122,6 @@ public class DockerInternalRuntime extends InternalRuntime<DockerRuntimeContext>
|
|||
dockerNetworkLifecycle.createNetwork(dockerEnvironment.getNetwork());
|
||||
|
||||
String machineName = startQueue.peek();
|
||||
DockerMachine dockerMachine;
|
||||
while (machineName != null) {
|
||||
DockerContainerConfig service = dockerEnvironment.getServices().get(machineName);
|
||||
checkStartInterruption();
|
||||
|
|
@ -112,7 +130,7 @@ public class DockerInternalRuntime extends InternalRuntime<DockerRuntimeContext>
|
|||
.withEventType(MachineStatus.STARTING)
|
||||
.withMachineName(machineName));
|
||||
try {
|
||||
dockerMachine = startMachine(machineName, service, startOptions, machineName.equals(devMachineName));
|
||||
startMachine(machineName, service, startOptions, machineName.equals(devMachineName));
|
||||
eventService.publish(DtoFactory.newDto(MachineStatusEvent.class)
|
||||
.withIdentity(DtoConverter.asDto(identity))
|
||||
.withEventType(MachineStatus.RUNNING)
|
||||
|
|
@ -125,13 +143,6 @@ public class DockerInternalRuntime extends InternalRuntime<DockerRuntimeContext>
|
|||
.withError(e.getMessage()));
|
||||
throw e;
|
||||
}
|
||||
checkStartInterruption();
|
||||
try {
|
||||
startSynchronizer.addMachine(machineName, dockerMachine);
|
||||
} catch (InfrastructureException e) {
|
||||
destroyMachineQuietly(machineName, dockerMachine);
|
||||
throw e;
|
||||
}
|
||||
startQueue.poll();
|
||||
machineName = startQueue.peek();
|
||||
}
|
||||
|
|
@ -158,10 +169,33 @@ public class DockerInternalRuntime extends InternalRuntime<DockerRuntimeContext>
|
|||
}
|
||||
}
|
||||
|
||||
private DockerMachine startMachine(String name,
|
||||
DockerContainerConfig containerConfig,
|
||||
Map<String, String> startOptions,
|
||||
boolean isDev) throws InfrastructureException {
|
||||
@Override
|
||||
protected void internalStop(Map<String, String> stopOptions) throws InfrastructureException {
|
||||
startSynchronizer.interruptStartThread();
|
||||
try {
|
||||
destroyRuntime(stopOptions);
|
||||
} finally {
|
||||
contextsStorage.remove((DockerRuntimeContext)getContext());
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<String, ? extends Machine> getInternalMachines() {
|
||||
return startSynchronizer.getMachines()
|
||||
.entrySet()
|
||||
.stream()
|
||||
.collect(toMap(Map.Entry::getKey, e -> new MachineImpl(e.getValue())));
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<String, String> getProperties() {
|
||||
return Collections.unmodifiableMap(properties);
|
||||
}
|
||||
|
||||
private void startMachine(String name,
|
||||
DockerContainerConfig containerConfig,
|
||||
Map<String, String> startOptions,
|
||||
boolean isDev) throws InfrastructureException {
|
||||
DockerMachine dockerMachine;
|
||||
// TODO property name
|
||||
final RuntimeIdentity identity = getContext().getIdentity();
|
||||
|
|
@ -200,8 +234,15 @@ public class DockerInternalRuntime extends InternalRuntime<DockerRuntimeContext>
|
|||
identity,
|
||||
isDev);
|
||||
}
|
||||
try {
|
||||
checkStartInterruption();
|
||||
startSynchronizer.addMachine(name, dockerMachine);
|
||||
} catch (InfrastructureException e) {
|
||||
destroyMachineQuietly(name, dockerMachine);
|
||||
throw e;
|
||||
}
|
||||
startAgents(name, dockerMachine);
|
||||
return dockerMachine;
|
||||
checkServersReadiness(name, dockerMachine);
|
||||
}
|
||||
|
||||
// TODO rework to agent launchers
|
||||
|
|
@ -210,10 +251,10 @@ public class DockerInternalRuntime extends InternalRuntime<DockerRuntimeContext>
|
|||
if (machineConfig == null) {
|
||||
throw new InfrastructureException("Machine %s is not found in internal machines config of RuntimeContext");
|
||||
}
|
||||
for (InternalMachineConfig.ResolvedAgent resolvedAgent : machineConfig.getAgents()) {
|
||||
for (AgentImpl agent : machineConfig.getAgents()) {
|
||||
Thread thread = new Thread(() -> {
|
||||
try {
|
||||
dockerMachine.exec(resolvedAgent.getScript(), MessageProcessor.getDevNull());
|
||||
dockerMachine.exec(agent.getScript(), MessageProcessor.getDevNull());
|
||||
} catch (InfrastructureException e) {
|
||||
LOG.error(e.getLocalizedMessage(), e);
|
||||
}
|
||||
|
|
@ -223,6 +264,80 @@ public class DockerInternalRuntime extends InternalRuntime<DockerRuntimeContext>
|
|||
}
|
||||
}
|
||||
|
||||
private void checkServersReadiness(String machineName, DockerMachine dockerMachine)
|
||||
throws InfrastructureException {
|
||||
for (Map.Entry<String, ServerImpl> serverEntry : dockerMachine.getServers().entrySet()) {
|
||||
String serverRef = serverEntry.getKey();
|
||||
ServerImpl server = serverEntry.getValue();
|
||||
|
||||
LOG.info("Checking server {} of machine {}", serverRef, machineName);
|
||||
checkServerReadiness(machineName, serverRef, server.getUrl());
|
||||
}
|
||||
}
|
||||
|
||||
// TODO rework checks to ping servers concurrently and timeouts each ping in case of network/server hanging
|
||||
private void checkServerReadiness(String machineName,
|
||||
String serverRef,
|
||||
String serverUrl)
|
||||
throws InfrastructureException {
|
||||
|
||||
if (!livenessChecksPaths.containsKey(serverRef)) {
|
||||
return;
|
||||
}
|
||||
String livenessCheckPath = livenessChecksPaths.get(serverRef);
|
||||
URL url;
|
||||
try {
|
||||
url = UriBuilder.fromUri(serverUrl)
|
||||
.replacePath(livenessCheckPath)
|
||||
.build()
|
||||
.toURL();
|
||||
} catch (MalformedURLException e) {
|
||||
throw new InternalInfrastructureException("Server " + serverRef +
|
||||
" URL is invalid. Error: " + e.getLocalizedMessage(), e);
|
||||
}
|
||||
// max start time 180 seconds
|
||||
long readinessDeadLine = System.currentTimeMillis() + 3000 * 60;
|
||||
while (System.currentTimeMillis() < readinessDeadLine) {
|
||||
LOG.info("Checking agent {} of machine {} at {}", serverRef, machineName,
|
||||
System.currentTimeMillis());
|
||||
checkStartInterruption();
|
||||
if (isHttpConnectionSucceed(url)) {
|
||||
// TODO protect with lock, from null, from exceptions
|
||||
DockerMachine machine = startSynchronizer.getMachines().get(machineName);
|
||||
machine.setServerStatus(serverRef, ServerStatus.RUNNING);
|
||||
eventService.publish(DtoFactory.newDto(ServerStatusEvent.class)
|
||||
.withIdentity(DtoConverter.asDto(identity))
|
||||
.withMachineName(machineName)
|
||||
.withServerName(serverRef)
|
||||
.withStatus(ServerStatus.RUNNING)
|
||||
.withServerUrl(serverUrl));
|
||||
LOG.info("Server {} of machine {} started", serverRef, machineName);
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
Thread.sleep(3000);
|
||||
} catch (InterruptedException e) {
|
||||
throw new InternalInfrastructureException("Interrupted");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private boolean isHttpConnectionSucceed(URL serverUrl) {
|
||||
HttpURLConnection httpURLConnection = null;
|
||||
try {
|
||||
httpURLConnection = (HttpURLConnection)serverUrl.openConnection();
|
||||
int responseCode = httpURLConnection.getResponseCode();
|
||||
return responseCode >= 200 && responseCode < 400;
|
||||
} catch (IOException e) {
|
||||
return false;
|
||||
} finally {
|
||||
if (httpURLConnection != null) {
|
||||
httpURLConnection.disconnect();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private DockerContainerConfig normalizeSource(DockerContainerConfig containerConfig,
|
||||
MachineSource machineSource) {
|
||||
DockerContainerConfig serviceWithNormalizedSource = new DockerContainerConfig(containerConfig);
|
||||
|
|
@ -247,29 +362,8 @@ public class DockerInternalRuntime extends InternalRuntime<DockerRuntimeContext>
|
|||
return serviceWithNormalizedSource;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void internalStop(Map<String, String> stopOptions) throws InfrastructureException {
|
||||
startSynchronizer.interruptStartThread();
|
||||
try {
|
||||
destroyRuntime(stopOptions);
|
||||
} finally {
|
||||
contextsStorage.remove((DockerRuntimeContext)getContext());
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<String, ? extends Machine> getInternalMachines() {
|
||||
return Collections.unmodifiableMap(startSynchronizer.getMachines());
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public Map<String, String> getProperties() {
|
||||
return Collections.unmodifiableMap(properties);
|
||||
}
|
||||
|
||||
private void checkStartInterruption() throws InfrastructureException {
|
||||
if (Thread.interrupted()) {
|
||||
if (Thread.currentThread().isInterrupted()) {
|
||||
throw new InfrastructureException("Docker infrastructure runtime start was interrupted");
|
||||
}
|
||||
}
|
||||
|
|
@ -378,8 +472,8 @@ public class DockerInternalRuntime extends InternalRuntime<DockerRuntimeContext>
|
|||
this.machines = new HashMap<>();
|
||||
}
|
||||
|
||||
public synchronized Map<String, ? extends Machine> getMachines() {
|
||||
return Collections.unmodifiableMap(machines);
|
||||
public synchronized Map<String, ? extends DockerMachine> getMachines() {
|
||||
return machines != null ? machines : Collections.emptyMap();
|
||||
}
|
||||
|
||||
public synchronized void addMachine(String name, DockerMachine machine) throws InternalInfrastructureException {
|
||||
|
|
|
|||
|
|
@ -13,7 +13,8 @@ package org.eclipse.che.workspace.infrastructure.docker;
|
|||
import com.google.inject.assistedinject.Assisted;
|
||||
|
||||
import org.eclipse.che.api.core.model.workspace.runtime.Machine;
|
||||
import org.eclipse.che.api.core.model.workspace.runtime.Server;
|
||||
import org.eclipse.che.api.core.model.workspace.runtime.ServerStatus;
|
||||
import org.eclipse.che.api.workspace.server.model.impl.ServerImpl;
|
||||
import org.eclipse.che.api.workspace.server.spi.InfrastructureException;
|
||||
import org.eclipse.che.api.workspace.server.spi.InternalInfrastructureException;
|
||||
import org.eclipse.che.commons.lang.NameGenerator;
|
||||
|
|
@ -96,6 +97,8 @@ public class DockerMachine implements Machine {
|
|||
private final ContainerInfo info;
|
||||
private final ServerEvaluationStrategyProvider provider;
|
||||
|
||||
private Map<String, ServerImpl> servers;
|
||||
|
||||
@Inject
|
||||
public DockerMachine(DockerConnector docker,
|
||||
String registry,
|
||||
|
|
@ -126,9 +129,23 @@ public class DockerMachine implements Machine {
|
|||
}
|
||||
|
||||
@Override
|
||||
public Map<String, ? extends Server> getServers() {
|
||||
ServerEvaluationStrategy strategy = provider.get();
|
||||
return strategy.getServers(info, "localhost", Collections.emptyMap());
|
||||
public Map<String, ServerImpl> getServers() {
|
||||
if(servers == null) {
|
||||
ServerEvaluationStrategy strategy = provider.get();
|
||||
servers = strategy.getServers(info, "localhost", Collections.emptyMap());
|
||||
}
|
||||
return servers;
|
||||
}
|
||||
|
||||
void setServerStatus(String serverRef, ServerStatus status) {
|
||||
if (servers == null) {
|
||||
throw new IllegalStateException("Servers are not initialized yet");
|
||||
}
|
||||
ServerImpl server = servers.get(serverRef);
|
||||
if (server == null) {
|
||||
throw new IllegalArgumentException("Server with provided reference " + serverRef + " missing");
|
||||
}
|
||||
server.setStatus(status);
|
||||
}
|
||||
|
||||
public void exec(String script, MessageProcessor<LogMessage> messageProcessor) throws InfrastructureException {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,775 @@
|
|||
/*******************************************************************************
|
||||
* Copyright (c) 2012-2017 Codenvy, S.A.
|
||||
* All rights reserved. This program and the accompanying materials
|
||||
* are made available under the terms of the Eclipse Public License v1.0
|
||||
* which accompanies this distribution, and is available at
|
||||
* http://www.eclipse.org/legal/epl-v10.html
|
||||
*
|
||||
* Contributors:
|
||||
* Codenvy, S.A. - initial API and implementation
|
||||
*******************************************************************************/
|
||||
package org.eclipse.che.workspace.infrastructure.docker;
|
||||
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
import com.google.common.base.MoreObjects;
|
||||
import com.google.common.base.Strings;
|
||||
import com.google.common.collect.ImmutableSet;
|
||||
|
||||
import org.eclipse.che.api.core.model.workspace.config.ServerConfig;
|
||||
import org.eclipse.che.api.core.model.workspace.runtime.RuntimeIdentity;
|
||||
import org.eclipse.che.api.core.util.FileCleaner;
|
||||
import org.eclipse.che.api.core.util.SystemInfo;
|
||||
import org.eclipse.che.api.workspace.server.model.impl.MachineSourceImpl;
|
||||
import org.eclipse.che.api.workspace.server.spi.InfrastructureException;
|
||||
import org.eclipse.che.api.workspace.server.spi.InternalInfrastructureException;
|
||||
import org.eclipse.che.commons.annotation.Nullable;
|
||||
import org.eclipse.che.commons.env.EnvironmentContext;
|
||||
import org.eclipse.che.commons.lang.os.WindowsPathEscaper;
|
||||
import org.eclipse.che.plugin.docker.client.DockerConnector;
|
||||
import org.eclipse.che.plugin.docker.client.ProgressMonitor;
|
||||
import org.eclipse.che.plugin.docker.client.UserSpecificDockerRegistryCredentialsProvider;
|
||||
import org.eclipse.che.plugin.docker.client.exception.ImageNotFoundException;
|
||||
import org.eclipse.che.plugin.docker.client.json.ContainerConfig;
|
||||
import org.eclipse.che.plugin.docker.client.json.ContainerInfo;
|
||||
import org.eclipse.che.plugin.docker.client.json.Filters;
|
||||
import org.eclipse.che.plugin.docker.client.json.HostConfig;
|
||||
import org.eclipse.che.plugin.docker.client.json.ImageConfig;
|
||||
import org.eclipse.che.plugin.docker.client.json.PortBinding;
|
||||
import org.eclipse.che.plugin.docker.client.json.Volume;
|
||||
import org.eclipse.che.plugin.docker.client.json.container.NetworkingConfig;
|
||||
import org.eclipse.che.plugin.docker.client.json.network.ConnectContainer;
|
||||
import org.eclipse.che.plugin.docker.client.json.network.EndpointConfig;
|
||||
import org.eclipse.che.plugin.docker.client.params.BuildImageParams;
|
||||
import org.eclipse.che.plugin.docker.client.params.CreateContainerParams;
|
||||
import org.eclipse.che.plugin.docker.client.params.ListImagesParams;
|
||||
import org.eclipse.che.plugin.docker.client.params.PullParams;
|
||||
import org.eclipse.che.plugin.docker.client.params.RemoveContainerParams;
|
||||
import org.eclipse.che.plugin.docker.client.params.RemoveImageParams;
|
||||
import org.eclipse.che.plugin.docker.client.params.StartContainerParams;
|
||||
import org.eclipse.che.plugin.docker.client.params.TagParams;
|
||||
import org.eclipse.che.plugin.docker.client.params.network.ConnectContainerToNetworkParams;
|
||||
import org.eclipse.che.workspace.infrastructure.docker.exception.SourceNotFoundException;
|
||||
import org.eclipse.che.workspace.infrastructure.docker.model.DockerContainerConfig;
|
||||
import org.eclipse.che.workspace.infrastructure.docker.monit.DockerMachineStopDetector;
|
||||
import org.eclipse.che.workspace.infrastructure.docker.strategy.ServerEvaluationStrategyProvider;
|
||||
import org.slf4j.Logger;
|
||||
|
||||
import javax.inject.Inject;
|
||||
import javax.inject.Named;
|
||||
import java.io.File;
|
||||
import java.io.FileWriter;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.function.Function;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import static java.lang.String.format;
|
||||
import static java.util.Collections.emptyMap;
|
||||
import static java.util.Collections.singletonList;
|
||||
import static java.util.Collections.singletonMap;
|
||||
import static java.util.stream.Collectors.toMap;
|
||||
import static java.util.stream.Collectors.toSet;
|
||||
import static org.eclipse.che.workspace.infrastructure.docker.DockerMachine.CHE_WORKSPACE_ID;
|
||||
import static org.eclipse.che.workspace.infrastructure.docker.DockerMachine.LATEST_TAG;
|
||||
import static org.eclipse.che.workspace.infrastructure.docker.DockerMachine.USER_TOKEN;
|
||||
import static org.slf4j.LoggerFactory.getLogger;
|
||||
|
||||
/**
|
||||
* @author Alexander Garagatyi
|
||||
*/
|
||||
public class DockerMachineStarter {
|
||||
private static final Logger LOG = getLogger(DockerMachineStarter.class);
|
||||
/**
|
||||
* Prefix of image repository, used to identify that the image is a machine saved to snapshot.
|
||||
*/
|
||||
public static final String MACHINE_SNAPSHOT_PREFIX = "machine_snapshot_";
|
||||
|
||||
public static final Pattern SNAPSHOT_LOCATION_PATTERN = Pattern.compile("(.+/)?" + MACHINE_SNAPSHOT_PREFIX + ".+");
|
||||
|
||||
private static final String CONTAINER_EXITED_ERROR = "We detected that a machine exited unexpectedly. " +
|
||||
"This may be caused by a container in interactive mode " +
|
||||
"or a container that requires additional arguments to start. " +
|
||||
"Please check the container recipe.";
|
||||
|
||||
// CMDs and entrypoints that lead to exiting of container right after start
|
||||
private static final Set<List<String>> badCMDs = ImmutableSet.of(singletonList("/bin/bash"),
|
||||
singletonList("/bin/sh"),
|
||||
singletonList("bash"),
|
||||
singletonList("sh"),
|
||||
Arrays.asList("/bin/sh", "-c", "/bin/sh"),
|
||||
Arrays.asList("/bin/sh", "-c",
|
||||
"/bin/bash"),
|
||||
Arrays.asList("/bin/sh", "-c", "bash"),
|
||||
Arrays.asList("/bin/sh", "-c", "sh"));
|
||||
|
||||
private static final Set<List<String>> badEntrypoints =
|
||||
ImmutableSet.<List<String>>builder().addAll(badCMDs)
|
||||
.add(Arrays.asList("/bin/sh", "-c"))
|
||||
.add(Arrays.asList("/bin/bash", "-c"))
|
||||
.add(Arrays.asList("sh", "-c"))
|
||||
.add(Arrays.asList("bash", "-c"))
|
||||
.build();
|
||||
|
||||
private final DockerConnector docker;
|
||||
private final UserSpecificDockerRegistryCredentialsProvider dockerCredentials;
|
||||
// TODO spi fix in #5102
|
||||
// private final ExecutorService executor;
|
||||
private final DockerMachineStopDetector dockerInstanceStopDetector;
|
||||
private final boolean doForcePullImage;
|
||||
private final boolean privilegedMode;
|
||||
private final int pidsLimit;
|
||||
private final List<String> devMachinePortsToExpose;
|
||||
private final List<String> commonMachinePortsToExpose;
|
||||
private final List<String> devMachineSystemVolumes;
|
||||
private final List<String> commonMachineSystemVolumes;
|
||||
private final Map<String, String> devMachineEnvVariables;
|
||||
private final Map<String, String> commonMachineEnvVariables;
|
||||
private final String[] allMachinesExtraHosts;
|
||||
private final boolean snapshotUseRegistry;
|
||||
private final double memorySwapMultiplier;
|
||||
private final Set<String> additionalNetworks;
|
||||
private final String parentCgroup;
|
||||
private final String cpusetCpus;
|
||||
private final String registry;
|
||||
private final String registryNamespace;
|
||||
private final long cpuPeriod;
|
||||
private final long cpuQuota;
|
||||
private final WindowsPathEscaper windowsPathEscaper;
|
||||
private final String[] dnsResolvers;
|
||||
private ServerEvaluationStrategyProvider serverEvaluationStrategyProvider;
|
||||
private final Map<String, String> buildArgs;
|
||||
|
||||
@Inject
|
||||
public DockerMachineStarter(DockerConnector docker,
|
||||
UserSpecificDockerRegistryCredentialsProvider dockerCredentials,
|
||||
DockerMachineStopDetector dockerMachineStopDetector,
|
||||
@Named("machine.docker.dev_machine.machine_servers") Set<ServerConfig> devMachineServers,
|
||||
@Named("machine.docker.machine_servers") Set<ServerConfig> allMachinesServers,
|
||||
@Named("machine.docker.dev_machine.machine_volumes") Set<String> devMachineSystemVolumes,
|
||||
@Named("machine.docker.machine_volumes") Set<String> allMachinesSystemVolumes,
|
||||
@Named("che.docker.always_pull_image") boolean doForcePullImage,
|
||||
@Named("che.docker.privileged") boolean privilegedMode,
|
||||
@Named("che.docker.pids_limit") int pidsLimit,
|
||||
@Named("che.docker.registry") String registry,
|
||||
@Named("che.docker.namespace") @Nullable String registryNamespace,
|
||||
@Named("machine.docker.dev_machine.machine_env") Set<String> devMachineEnvVariables,
|
||||
@Named("machine.docker.machine_env") Set<String> allMachinesEnvVariables,
|
||||
@Named("che.docker.registry_for_snapshots") boolean snapshotUseRegistry,
|
||||
@Named("che.docker.swap") double memorySwapMultiplier,
|
||||
@Named("machine.docker.networks") Set<Set<String>> additionalNetworks,
|
||||
@Nullable @Named("che.docker.parent_cgroup") String parentCgroup,
|
||||
@Nullable @Named("che.docker.cpuset_cpus") String cpusetCpus,
|
||||
@Named("che.docker.cpu_period") long cpuPeriod,
|
||||
@Named("che.docker.cpu_quota") long cpuQuota,
|
||||
WindowsPathEscaper windowsPathEscaper,
|
||||
@Named("che.docker.extra_hosts") Set<Set<String>> additionalHosts,
|
||||
@Nullable @Named("che.docker.dns_resolvers") String[] dnsResolvers,
|
||||
ServerEvaluationStrategyProvider serverEvaluationStrategyProvider,
|
||||
@Named("che.docker.build_args") Map<String, String> buildArgs) {
|
||||
// TODO spi should we move all configuration stuff into infrastructure provisioner and left logic of container start here only
|
||||
this.docker = docker;
|
||||
this.dockerCredentials = dockerCredentials;
|
||||
this.dockerInstanceStopDetector = dockerMachineStopDetector;
|
||||
this.doForcePullImage = doForcePullImage;
|
||||
this.privilegedMode = privilegedMode;
|
||||
this.snapshotUseRegistry = snapshotUseRegistry;
|
||||
// use-cases:
|
||||
// -1 enable unlimited swap
|
||||
// 0 disable swap
|
||||
// 0.5 enable swap with size equal to half of current memory size
|
||||
// 1 enable swap with size equal to current memory size
|
||||
//
|
||||
// according to docker docs field memorySwap should be equal to memory+swap
|
||||
// we calculate this field as memorySwap=memory * (1 + multiplier) so we just add 1 to multiplier
|
||||
this.memorySwapMultiplier = memorySwapMultiplier == -1 ? -1 : memorySwapMultiplier + 1;
|
||||
this.parentCgroup = parentCgroup;
|
||||
this.cpusetCpus = cpusetCpus;
|
||||
this.cpuPeriod = cpuPeriod;
|
||||
this.cpuQuota = cpuQuota;
|
||||
this.windowsPathEscaper = windowsPathEscaper;
|
||||
this.registryNamespace = registryNamespace;
|
||||
this.registry = registry;
|
||||
this.pidsLimit = pidsLimit;
|
||||
this.dnsResolvers = dnsResolvers;
|
||||
this.buildArgs = buildArgs;
|
||||
this.serverEvaluationStrategyProvider = serverEvaluationStrategyProvider;
|
||||
|
||||
allMachinesSystemVolumes = removeEmptyAndNullValues(allMachinesSystemVolumes);
|
||||
devMachineSystemVolumes = removeEmptyAndNullValues(devMachineSystemVolumes);
|
||||
|
||||
allMachinesSystemVolumes = allMachinesSystemVolumes.stream()
|
||||
.map(line -> line.split(";"))
|
||||
.flatMap(Arrays::stream)
|
||||
.distinct()
|
||||
.collect(toSet());
|
||||
|
||||
devMachineSystemVolumes = devMachineSystemVolumes.stream()
|
||||
.map(line -> line.split(";"))
|
||||
.flatMap(Arrays::stream)
|
||||
.distinct()
|
||||
.collect(toSet());
|
||||
|
||||
if (SystemInfo.isWindows()) {
|
||||
allMachinesSystemVolumes = escapePaths(allMachinesSystemVolumes);
|
||||
devMachineSystemVolumes = escapePaths(devMachineSystemVolumes);
|
||||
}
|
||||
this.commonMachineSystemVolumes = new ArrayList<>(allMachinesSystemVolumes);
|
||||
List<String> devMachineVolumes = new ArrayList<>(allMachinesSystemVolumes.size()
|
||||
+ devMachineSystemVolumes.size());
|
||||
devMachineVolumes.addAll(allMachinesSystemVolumes);
|
||||
devMachineVolumes.addAll(devMachineSystemVolumes);
|
||||
this.devMachineSystemVolumes = devMachineVolumes;
|
||||
|
||||
this.devMachinePortsToExpose = new ArrayList<>(allMachinesServers.size() + devMachineServers.size());
|
||||
this.commonMachinePortsToExpose = new ArrayList<>(allMachinesServers.size());
|
||||
for (ServerConfig serverConf : devMachineServers) {
|
||||
devMachinePortsToExpose.add(serverConf.getPort());
|
||||
}
|
||||
for (ServerConfig serverConf : allMachinesServers) {
|
||||
commonMachinePortsToExpose.add(serverConf.getPort());
|
||||
devMachinePortsToExpose.add(serverConf.getPort());
|
||||
}
|
||||
|
||||
allMachinesEnvVariables = removeEmptyAndNullValues(allMachinesEnvVariables);
|
||||
devMachineEnvVariables = removeEmptyAndNullValues(devMachineEnvVariables);
|
||||
this.commonMachineEnvVariables = new HashMap<>();
|
||||
this.devMachineEnvVariables = new HashMap<>();
|
||||
allMachinesEnvVariables.forEach(envVar -> {
|
||||
String[] split = envVar.split("=", 2);
|
||||
this.commonMachineEnvVariables.put(split[0], split[1]);
|
||||
this.devMachineEnvVariables.put(split[0], split[1]);
|
||||
});
|
||||
devMachineEnvVariables.forEach(envVar -> {
|
||||
String[] split = envVar.split("=", 2);
|
||||
this.devMachineEnvVariables.put(split[0], split[1]);
|
||||
});
|
||||
|
||||
this.allMachinesExtraHosts = additionalHosts.stream()
|
||||
.flatMap(Set::stream)
|
||||
.toArray(String[]::new);
|
||||
|
||||
this.additionalNetworks = additionalNetworks.stream()
|
||||
.flatMap(Set::stream)
|
||||
.collect(toSet());
|
||||
|
||||
// TODO spi fix in #5102
|
||||
// single point of failure in case of highly loaded system
|
||||
/*executor = Executors.newCachedThreadPool(new ThreadFactoryBuilder().setNameFormat("MachineLogsStreamer-%d")
|
||||
.setUncaughtExceptionHandler(
|
||||
LoggingUncaughtExceptionHandler
|
||||
.getInstance())
|
||||
.setDaemon(true)
|
||||
.build());*/
|
||||
}
|
||||
|
||||
/**
|
||||
* Start Docker machine by performing all needed operations such as pull, build, create container, etc.
|
||||
*
|
||||
* @param networkName
|
||||
* name of a network Docker container should use
|
||||
* @param machineName
|
||||
* name of Docker machine
|
||||
* @param containerConfig
|
||||
* configuration of container to start
|
||||
* @param identity
|
||||
* identity of user that starts machine
|
||||
* @param isDev
|
||||
* whether machine is dev or not
|
||||
* @return {@link DockerMachine} instance that represents started container
|
||||
* @throws InternalInfrastructureException
|
||||
* if internal error occurs
|
||||
* @throws SourceNotFoundException
|
||||
* if image for container creation is missing
|
||||
* @throws InfrastructureException
|
||||
* if any other error occurs
|
||||
*/
|
||||
public DockerMachine startService(String networkName,
|
||||
String machineName,
|
||||
DockerContainerConfig containerConfig,
|
||||
RuntimeIdentity identity,
|
||||
boolean isDev) throws InfrastructureException {
|
||||
String workspaceId = identity.getWorkspaceId();
|
||||
|
||||
// copy to not affect/be affected by changes in origin
|
||||
containerConfig = new DockerContainerConfig(containerConfig);
|
||||
|
||||
// TODO spi fix in #5102
|
||||
ProgressMonitor progressMonitor = ProgressMonitor.DEV_NULL;
|
||||
/*LineConsumer machineLogger = new ListLineConsumer();
|
||||
ProgressLineFormatterImpl progressLineFormatter = new ProgressLineFormatterImpl();
|
||||
ProgressMonitor progressMonitor = currentProgressStatus -> {
|
||||
try {
|
||||
machineLogger.writeLine(progressLineFormatter.format(currentProgressStatus));
|
||||
} catch (IOException e) {
|
||||
LOG.error(e.getLocalizedMessage(), e);
|
||||
}
|
||||
};*/
|
||||
|
||||
String container = null;
|
||||
try {
|
||||
String image = prepareImage(machineName,
|
||||
containerConfig,
|
||||
progressMonitor);
|
||||
|
||||
container = createContainer(workspaceId,
|
||||
machineName,
|
||||
isDev,
|
||||
image,
|
||||
networkName,
|
||||
containerConfig);
|
||||
|
||||
connectContainerToAdditionalNetworks(container,
|
||||
containerConfig);
|
||||
|
||||
docker.startContainer(StartContainerParams.create(container));
|
||||
|
||||
checkContainerIsRunning(container);
|
||||
|
||||
// TODO spi fix in #5102
|
||||
/*readContainerLogsInSeparateThread(container,
|
||||
workspaceId,
|
||||
service.getId(),
|
||||
machineLogger);*/
|
||||
|
||||
dockerInstanceStopDetector.startDetection(container,
|
||||
containerConfig.getId(),
|
||||
workspaceId);
|
||||
|
||||
return new DockerMachine(docker,
|
||||
registry,
|
||||
registryNamespace,
|
||||
snapshotUseRegistry,
|
||||
container,
|
||||
image,
|
||||
serverEvaluationStrategyProvider,
|
||||
dockerInstanceStopDetector);
|
||||
} catch (RuntimeException | IOException | InfrastructureException e) {
|
||||
cleanUpContainer(container);
|
||||
if (e instanceof InfrastructureException) {
|
||||
throw (InfrastructureException)e;
|
||||
} else {
|
||||
throw new InternalInfrastructureException(e.getLocalizedMessage(), e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private String prepareImage(String machineName,
|
||||
DockerContainerConfig service,
|
||||
ProgressMonitor progressMonitor)
|
||||
throws SourceNotFoundException,
|
||||
InternalInfrastructureException {
|
||||
|
||||
String imageName = "eclipse-che/" + service.getContainerName();
|
||||
if ((service.getBuild() == null || (service.getBuild().getContext() == null &&
|
||||
service.getBuild().getDockerfileContent() == null)) &&
|
||||
service.getImage() == null) {
|
||||
|
||||
throw new InternalInfrastructureException(
|
||||
format("Che service '%s' doesn't have neither build nor image fields", machineName));
|
||||
}
|
||||
|
||||
if (service.getBuild() != null && (service.getBuild().getContext() != null ||
|
||||
service.getBuild().getDockerfileContent() != null)) {
|
||||
buildImage(service, imageName, doForcePullImage, progressMonitor);
|
||||
} else {
|
||||
pullImage(service, imageName, progressMonitor);
|
||||
}
|
||||
|
||||
return imageName;
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds Docker image for container creation.
|
||||
*
|
||||
* @param containerConfig
|
||||
* configuration of container
|
||||
* @param machineImageName
|
||||
* name of image that should be applied to built image
|
||||
* @param doForcePullOnBuild
|
||||
* whether re-pulling of base image should be performed when it exists locally
|
||||
* @param progressMonitor
|
||||
* consumer of build logs
|
||||
* @throws InternalInfrastructureException
|
||||
* when any error occurs
|
||||
*/
|
||||
protected void buildImage(DockerContainerConfig containerConfig,
|
||||
String machineImageName,
|
||||
boolean doForcePullOnBuild,
|
||||
ProgressMonitor progressMonitor)
|
||||
throws InternalInfrastructureException {
|
||||
|
||||
File workDir = null;
|
||||
try {
|
||||
BuildImageParams buildImageParams;
|
||||
if (containerConfig.getBuild() != null &&
|
||||
containerConfig.getBuild().getDockerfileContent() != null) {
|
||||
|
||||
workDir = Files.createTempDirectory(null).toFile();
|
||||
final File dockerfileFile = new File(workDir, "Dockerfile");
|
||||
try (FileWriter output = new FileWriter(dockerfileFile)) {
|
||||
output.append(containerConfig.getBuild().getDockerfileContent());
|
||||
}
|
||||
|
||||
buildImageParams = BuildImageParams.create(dockerfileFile);
|
||||
} else {
|
||||
buildImageParams = BuildImageParams.create(containerConfig.getBuild().getContext())
|
||||
.withDockerfile(containerConfig.getBuild().getDockerfilePath());
|
||||
}
|
||||
Map<String, String> buildArgs;
|
||||
if (containerConfig.getBuild().getArgs() == null || containerConfig.getBuild().getArgs().isEmpty()) {
|
||||
buildArgs = this.buildArgs;
|
||||
} else {
|
||||
buildArgs = new HashMap<>(this.buildArgs);
|
||||
buildArgs.putAll(containerConfig.getBuild().getArgs());
|
||||
}
|
||||
buildImageParams.withForceRemoveIntermediateContainers(true)
|
||||
.withRepository(machineImageName)
|
||||
.withAuthConfigs(dockerCredentials.getCredentials())
|
||||
.withDoForcePull(doForcePullOnBuild)
|
||||
.withMemoryLimit(containerConfig.getMemLimit())
|
||||
.withMemorySwapLimit(-1)
|
||||
.withCpusetCpus(cpusetCpus)
|
||||
.withCpuPeriod(cpuPeriod)
|
||||
.withCpuQuota(cpuQuota)
|
||||
.withBuildArgs(buildArgs);
|
||||
|
||||
docker.buildImage(buildImageParams, progressMonitor);
|
||||
} catch (IOException e) {
|
||||
throw new InternalInfrastructureException(e.getLocalizedMessage(), e);
|
||||
} finally {
|
||||
if (workDir != null) {
|
||||
FileCleaner.addFile(workDir);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Pulls docker image for container creation.
|
||||
*
|
||||
* @param service
|
||||
* service that provides description of image that should be pulled
|
||||
* @param machineImageName
|
||||
* name of the image that should be assigned on pull
|
||||
* @param progressMonitor
|
||||
* consumer of output
|
||||
* @throws SourceNotFoundException
|
||||
* if image for pulling not found in registry
|
||||
* @throws InternalInfrastructureException
|
||||
* if any other error occurs
|
||||
*/
|
||||
protected void pullImage(DockerContainerConfig service,
|
||||
String machineImageName,
|
||||
ProgressMonitor progressMonitor) throws InternalInfrastructureException,
|
||||
SourceNotFoundException {
|
||||
DockerMachineSource dockerMachineSource = new DockerMachineSource(
|
||||
new MachineSourceImpl("image").setLocation(service.getImage()));
|
||||
if (dockerMachineSource.getRepository() == null) {
|
||||
throw new InternalInfrastructureException(
|
||||
format("Machine creation failed. Machine source is invalid. No repository is defined. Found '%s'.",
|
||||
dockerMachineSource));
|
||||
}
|
||||
|
||||
try {
|
||||
boolean isSnapshot = SNAPSHOT_LOCATION_PATTERN.matcher(dockerMachineSource.getLocation()).matches();
|
||||
boolean isImageExistLocally = isDockerImageExistLocally(dockerMachineSource.getRepository());
|
||||
if ((!isSnapshot && (doForcePullImage || !isImageExistLocally)) || (isSnapshot && snapshotUseRegistry)) {
|
||||
PullParams pullParams = PullParams.create(dockerMachineSource.getRepository())
|
||||
.withTag(MoreObjects.firstNonNull(dockerMachineSource.getTag(),
|
||||
LATEST_TAG))
|
||||
.withRegistry(dockerMachineSource.getRegistry())
|
||||
.withAuthConfigs(dockerCredentials.getCredentials());
|
||||
docker.pull(pullParams, progressMonitor);
|
||||
}
|
||||
|
||||
String fullNameOfPulledImage = dockerMachineSource.getLocation(false);
|
||||
try {
|
||||
// tag image with generated name to allow sysadmin recognize it
|
||||
docker.tag(TagParams.create(fullNameOfPulledImage, machineImageName));
|
||||
} catch (ImageNotFoundException nfEx) {
|
||||
throw new SourceNotFoundException(nfEx.getLocalizedMessage(), nfEx);
|
||||
}
|
||||
|
||||
// remove unneeded tag if restoring snapshot from registry
|
||||
if (isSnapshot && snapshotUseRegistry) {
|
||||
docker.removeImage(RemoveImageParams.create(fullNameOfPulledImage).withForce(false));
|
||||
}
|
||||
} catch (IOException e) {
|
||||
throw new InternalInfrastructureException("Can't create machine from image. Cause: " +
|
||||
e.getLocalizedMessage(), e);
|
||||
}
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
boolean isDockerImageExistLocally(String imageName) {
|
||||
try {
|
||||
return !docker.listImages(ListImagesParams.create()
|
||||
.withFilters(new Filters().withFilter("reference", imageName)))
|
||||
.isEmpty();
|
||||
} catch (IOException e) {
|
||||
LOG.warn("Failed to check image {} availability. Cause: {}", imageName, e.getMessage(), e);
|
||||
return false; // consider that image doesn't exist locally
|
||||
}
|
||||
}
|
||||
|
||||
private String createContainer(String workspaceId,
|
||||
String machineName,
|
||||
boolean isDev,
|
||||
String image,
|
||||
String networkName,
|
||||
DockerContainerConfig containerConfig) throws IOException {
|
||||
|
||||
long machineMemorySwap = memorySwapMultiplier == -1 ?
|
||||
-1 :
|
||||
(long)(containerConfig.getMemLimit() * memorySwapMultiplier);
|
||||
|
||||
addSystemWideContainerSettings(workspaceId,
|
||||
isDev,
|
||||
containerConfig);
|
||||
|
||||
EndpointConfig endpointConfig = new EndpointConfig().withAliases(machineName)
|
||||
.withLinks(toArrayIfNotNull(containerConfig.getLinks()));
|
||||
NetworkingConfig networkingConfig = new NetworkingConfig().withEndpointsConfig(singletonMap(networkName,
|
||||
endpointConfig));
|
||||
|
||||
HostConfig hostConfig = new HostConfig();
|
||||
hostConfig.withMemorySwap(machineMemorySwap)
|
||||
.withMemory(containerConfig.getMemLimit())
|
||||
.withNetworkMode(networkName)
|
||||
.withLinks(toArrayIfNotNull(containerConfig.getLinks()))
|
||||
.withPortBindings(containerConfig.getPorts()
|
||||
.stream()
|
||||
.collect(toMap(Function.identity(), value -> new PortBinding[0])))
|
||||
.withVolumesFrom(toArrayIfNotNull(containerConfig.getVolumesFrom()));
|
||||
|
||||
ContainerConfig config = new ContainerConfig();
|
||||
config.withImage(image)
|
||||
.withExposedPorts(containerConfig.getExpose()
|
||||
.stream()
|
||||
.distinct()
|
||||
.collect(toMap(Function.identity(), value -> emptyMap())))
|
||||
.withHostConfig(hostConfig)
|
||||
.withCmd(toArrayIfNotNull(containerConfig.getCommand()))
|
||||
.withEntrypoint(toArrayIfNotNull(containerConfig.getEntrypoint()))
|
||||
.withLabels(containerConfig.getLabels())
|
||||
.withNetworkingConfig(networkingConfig)
|
||||
.withEnv(containerConfig.getEnvironment()
|
||||
.entrySet()
|
||||
.stream()
|
||||
.map(entry -> entry.getKey() + "=" + entry.getValue())
|
||||
.toArray(String[]::new));
|
||||
|
||||
List<String> bindMountVolumes = new ArrayList<>();
|
||||
Map<String, Volume> nonBindMountVolumes = new HashMap<>();
|
||||
for (String volume : containerConfig.getVolumes()) {
|
||||
// If volume contains colon then it is bind volume, otherwise - non bind-mount volume.
|
||||
if (volume.contains(":")) {
|
||||
bindMountVolumes.add(volume);
|
||||
} else {
|
||||
nonBindMountVolumes.put(volume, new Volume());
|
||||
}
|
||||
}
|
||||
hostConfig.setBinds(bindMountVolumes.toArray(new String[bindMountVolumes.size()]));
|
||||
config.setVolumes(nonBindMountVolumes);
|
||||
|
||||
addStaticDockerConfiguration(config);
|
||||
|
||||
setNonExitingContainerCommandIfNeeded(config);
|
||||
|
||||
return docker.createContainer(CreateContainerParams.create(config)
|
||||
.withContainerName(containerConfig.getContainerName()))
|
||||
.getId();
|
||||
}
|
||||
|
||||
private void addStaticDockerConfiguration(ContainerConfig config) {
|
||||
config.getHostConfig()
|
||||
.withPidsLimit(pidsLimit)
|
||||
.withExtraHosts(allMachinesExtraHosts)
|
||||
.withPrivileged(privilegedMode)
|
||||
.withPublishAllPorts(true)
|
||||
.withDns(dnsResolvers);
|
||||
// CPU limits
|
||||
config.getHostConfig()
|
||||
.withCpusetCpus(cpusetCpus)
|
||||
.withCpuQuota(cpuQuota)
|
||||
.withCpuPeriod(cpuPeriod);
|
||||
// Cgroup parent for custom limits
|
||||
config.getHostConfig().setCgroupParent(parentCgroup);
|
||||
}
|
||||
|
||||
private void addSystemWideContainerSettings(String workspaceId,
|
||||
boolean isDev,
|
||||
DockerContainerConfig containerConfig) {
|
||||
List<String> portsToExpose;
|
||||
List<String> volumes;
|
||||
Map<String, String> env;
|
||||
if (isDev) {
|
||||
portsToExpose = devMachinePortsToExpose;
|
||||
volumes = devMachineSystemVolumes;
|
||||
env = new HashMap<>(devMachineEnvVariables);
|
||||
env.put(CHE_WORKSPACE_ID, workspaceId);
|
||||
env.put(USER_TOKEN, getUserToken(workspaceId));
|
||||
} else {
|
||||
portsToExpose = commonMachinePortsToExpose;
|
||||
env = commonMachineEnvVariables;
|
||||
volumes = commonMachineSystemVolumes;
|
||||
}
|
||||
containerConfig.getExpose().addAll(portsToExpose);
|
||||
containerConfig.getEnvironment().putAll(env);
|
||||
containerConfig.getVolumes().addAll(volumes);
|
||||
containerConfig.getNetworks().addAll(additionalNetworks);
|
||||
}
|
||||
|
||||
// We can detect certain situation when container exited right after start.
|
||||
// We can detect
|
||||
// - when no command/entrypoint is set
|
||||
// - when most common shell interpreters are used and require additional arguments
|
||||
// - when most common shell interpreters are used and they require interactive mode which we don't support
|
||||
// When we identify such situation we change CMD/entrypoint in such a way that it runs "tail -f /dev/null".
|
||||
// This command does nothing and lasts until workspace is stopped.
|
||||
// Images such as "ubuntu" or "openjdk" fits this situation.
|
||||
protected void setNonExitingContainerCommandIfNeeded(ContainerConfig containerConfig) throws IOException {
|
||||
ImageConfig imageConfig = docker.inspectImage(containerConfig.getImage()).getConfig();
|
||||
List<String> cmd = imageConfig.getCmd() == null ?
|
||||
null : Arrays.asList(imageConfig.getCmd());
|
||||
List<String> entrypoint = imageConfig.getEntrypoint() == null ?
|
||||
null : Arrays.asList(imageConfig.getEntrypoint());
|
||||
|
||||
if ((entrypoint == null || badEntrypoints.contains(entrypoint)) && (cmd == null || badCMDs.contains(cmd))) {
|
||||
containerConfig.setCmd("tail", "-f", "/dev/null");
|
||||
containerConfig.setEntrypoint((String[])null);
|
||||
}
|
||||
}
|
||||
|
||||
// Inspect container right after start to check if it is running,
|
||||
// otherwise throw error that command should not exit right after container start
|
||||
protected void checkContainerIsRunning(String container) throws IOException, InfrastructureException {
|
||||
ContainerInfo containerInfo = docker.inspectContainer(container);
|
||||
if ("exited".equals(containerInfo.getState().getStatus())) {
|
||||
throw new InfrastructureException(CONTAINER_EXITED_ERROR);
|
||||
}
|
||||
}
|
||||
|
||||
// TODO spi fix in #5102
|
||||
/*private void readContainerLogsInSeparateThread(String container,
|
||||
String workspaceId,
|
||||
String machineId,
|
||||
LineConsumer outputConsumer) {
|
||||
executor.execute(() -> {
|
||||
long lastProcessedLogDate = 0;
|
||||
boolean isContainerRunning = true;
|
||||
int errorsCounter = 0;
|
||||
long lastErrorTime = 0;
|
||||
while (isContainerRunning) {
|
||||
try {
|
||||
docker.getContainerLogs(GetContainerLogsParams.create(container)
|
||||
.withFollow(true)
|
||||
.withSince(lastProcessedLogDate),
|
||||
new LogMessagePrinter(outputConsumer));
|
||||
isContainerRunning = false;
|
||||
} catch (SocketTimeoutException ste) {
|
||||
lastProcessedLogDate = System.currentTimeMillis() / 1000L;
|
||||
// reconnect to container
|
||||
} catch (ContainerNotFoundException e) {
|
||||
isContainerRunning = false;
|
||||
} catch (IOException e) {
|
||||
long errorTime = System.currentTimeMillis();
|
||||
lastProcessedLogDate = errorTime / 1000L;
|
||||
LOG.warn("Failed to get logs from machine {} of workspace {} backed by container {}, because: {}.",
|
||||
machineId,
|
||||
workspaceId,
|
||||
container,
|
||||
e.getMessage(),
|
||||
e);
|
||||
if (errorTime - lastErrorTime <
|
||||
20_000L) { // if new error occurs less than 20 seconds after previous
|
||||
if (++errorsCounter == 5) {
|
||||
LOG.error(
|
||||
"Too many errors while streaming logs from machine {} of workspace {} backed by container {}. " +
|
||||
"Logs streaming is closed. Last error: {}.",
|
||||
machineId,
|
||||
workspaceId,
|
||||
container,
|
||||
e.getMessage(),
|
||||
e);
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
errorsCounter = 1;
|
||||
}
|
||||
lastErrorTime = errorTime;
|
||||
|
||||
try {
|
||||
sleep(1_000);
|
||||
} catch (InterruptedException ie) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}*/
|
||||
|
||||
private void cleanUpContainer(String containerId) {
|
||||
try {
|
||||
if (containerId != null) {
|
||||
docker.removeContainer(RemoveContainerParams.create(containerId)
|
||||
.withRemoveVolumes(true)
|
||||
.withForce(true));
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
LOG.error("Failed to remove docker container {}", containerId, ex);
|
||||
}
|
||||
}
|
||||
|
||||
/** Converts list to array if it is not null, otherwise returns null */
|
||||
private String[] toArrayIfNotNull(List<String> list) {
|
||||
if (list == null) {
|
||||
return null;
|
||||
}
|
||||
return list.toArray(new String[list.size()]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns set that contains all non empty and non nullable values from specified set
|
||||
*/
|
||||
protected Set<String> removeEmptyAndNullValues(Set<String> paths) {
|
||||
return paths.stream()
|
||||
.filter(path -> !Strings.isNullOrEmpty(path))
|
||||
.collect(toSet());
|
||||
}
|
||||
|
||||
// workspaceId parameter is required, because in case of separate storage for tokens
|
||||
// you need to know exactly which workspace and which user to apply the token.
|
||||
protected String getUserToken(String wsId) {
|
||||
return EnvironmentContext.getCurrent().getSubject().getToken();
|
||||
}
|
||||
|
||||
/**
|
||||
* Escape paths for Windows system with boot@docker according to rules given here :
|
||||
* https://github.com/boot2docker/boot2docker/blob/master/README.md#virtualbox-guest-additions
|
||||
*
|
||||
* @param paths
|
||||
* set of paths to escape
|
||||
* @return set of escaped path
|
||||
*/
|
||||
private Set<String> escapePaths(Set<String> paths) {
|
||||
return paths.stream()
|
||||
.map(windowsPathEscaper::escapePath)
|
||||
.collect(toSet());
|
||||
}
|
||||
|
||||
// TODO spi should we move it into network lifecycle?
|
||||
private void connectContainerToAdditionalNetworks(String container,
|
||||
DockerContainerConfig service) throws IOException {
|
||||
|
||||
for (String network : service.getNetworks()) {
|
||||
docker.connectContainerToNetwork(
|
||||
ConnectContainerToNetworkParams.create(network, new ConnectContainer().withContainer(container)));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -17,22 +17,16 @@ import org.eclipse.che.api.agent.server.impl.AgentSorter;
|
|||
import org.eclipse.che.api.core.ValidationException;
|
||||
import org.eclipse.che.api.core.model.workspace.config.Environment;
|
||||
import org.eclipse.che.api.core.model.workspace.runtime.RuntimeIdentity;
|
||||
import org.eclipse.che.api.core.notification.EventService;
|
||||
import org.eclipse.che.api.workspace.server.URLRewriter;
|
||||
import org.eclipse.che.api.workspace.server.spi.InfrastructureException;
|
||||
import org.eclipse.che.api.workspace.server.spi.InternalRuntime;
|
||||
import org.eclipse.che.api.workspace.server.spi.RuntimeContext;
|
||||
import org.eclipse.che.api.workspace.shared.Utils;
|
||||
import org.eclipse.che.workspace.infrastructure.docker.model.DockerEnvironment;
|
||||
import org.eclipse.che.workspace.infrastructure.docker.snapshot.SnapshotDao;
|
||||
import org.slf4j.Logger;
|
||||
|
||||
import javax.inject.Inject;
|
||||
import java.net.URL;
|
||||
import java.util.List;
|
||||
|
||||
import static org.slf4j.LoggerFactory.getLogger;
|
||||
|
||||
/**
|
||||
* @author Alexander Garagatyi
|
||||
*/
|
||||
|
|
@ -48,18 +42,10 @@ import static org.slf4j.LoggerFactory.getLogger;
|
|||
// TODO Check if interruption came from stop or because of another reason
|
||||
// TODO if because of another reason stop environment
|
||||
public class DockerRuntimeContext extends RuntimeContext {
|
||||
private static final Logger LOG = getLogger(DockerRuntimeContext.class);
|
||||
|
||||
private final NetworkLifecycle dockerNetworkLifecycle;
|
||||
private final MachineStarter serviceStarter;
|
||||
private final DockerEnvironment dockerEnvironment;
|
||||
private final URLRewriter urlRewriter;
|
||||
private final DockerRuntimeFactory dockerRuntimeFactory;
|
||||
private final List<String> orderedServices;
|
||||
private final String devMachineName;
|
||||
private final ContextsStorage contextsStorage;
|
||||
private final SnapshotDao snapshotDao;
|
||||
private final DockerRegistryClient dockerRegistryClient;
|
||||
private final EventService eventService;
|
||||
|
||||
@Inject
|
||||
public DockerRuntimeContext(@Assisted DockerRuntimeInfrastructure infrastructure,
|
||||
|
|
@ -67,27 +53,15 @@ public class DockerRuntimeContext extends RuntimeContext {
|
|||
@Assisted Environment environment,
|
||||
@Assisted DockerEnvironment dockerEnvironment,
|
||||
@Assisted List<String> orderedServices,
|
||||
NetworkLifecycle dockerNetworkLifecycle,
|
||||
MachineStarter serviceStarter,
|
||||
URLRewriter urlRewriter,
|
||||
AgentSorter agentSorter,
|
||||
AgentRegistry agentRegistry,
|
||||
ContextsStorage contextsStorage,
|
||||
SnapshotDao snapshotDao,
|
||||
DockerRegistryClient dockerRegistryClient,
|
||||
EventService eventService)
|
||||
DockerRuntimeFactory dockerRuntimeFactory)
|
||||
throws ValidationException, InfrastructureException {
|
||||
super(environment, identity, infrastructure, agentSorter, agentRegistry);
|
||||
this.devMachineName = Utils.getDevMachineName(environment);
|
||||
this.orderedServices = orderedServices;
|
||||
this.dockerEnvironment = dockerEnvironment;
|
||||
this.dockerNetworkLifecycle = dockerNetworkLifecycle;
|
||||
this.serviceStarter = serviceStarter;
|
||||
this.urlRewriter = urlRewriter;
|
||||
this.contextsStorage = contextsStorage;
|
||||
this.snapshotDao = snapshotDao;
|
||||
this.dockerRegistryClient = dockerRegistryClient;
|
||||
this.eventService = eventService;
|
||||
this.dockerRuntimeFactory = dockerRuntimeFactory;
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -97,22 +71,10 @@ public class DockerRuntimeContext extends RuntimeContext {
|
|||
|
||||
@Override
|
||||
public InternalRuntime getRuntime() {
|
||||
return getInternalRuntime(); //TODO: instance field?
|
||||
|
||||
}
|
||||
|
||||
private InternalRuntime getInternalRuntime() {
|
||||
return new DockerInternalRuntime(this,
|
||||
devMachineName,
|
||||
urlRewriter,
|
||||
orderedServices,
|
||||
contextsStorage,
|
||||
dockerEnvironment,
|
||||
dockerNetworkLifecycle,
|
||||
serviceStarter,
|
||||
snapshotDao,
|
||||
dockerRegistryClient,
|
||||
identity,
|
||||
eventService);
|
||||
return dockerRuntimeFactory.createRuntime(this,
|
||||
devMachineName,
|
||||
orderedServices,
|
||||
dockerEnvironment,
|
||||
identity);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,29 @@
|
|||
/*******************************************************************************
|
||||
* Copyright (c) 2012-2017 Codenvy, S.A.
|
||||
* All rights reserved. This program and the accompanying materials
|
||||
* are made available under the terms of the Eclipse Public License v1.0
|
||||
* which accompanies this distribution, and is available at
|
||||
* http://www.eclipse.org/legal/epl-v10.html
|
||||
*
|
||||
* Contributors:
|
||||
* Codenvy, S.A. - initial API and implementation
|
||||
*******************************************************************************/
|
||||
package org.eclipse.che.workspace.infrastructure.docker;
|
||||
|
||||
import com.google.inject.assistedinject.Assisted;
|
||||
|
||||
import org.eclipse.che.api.core.model.workspace.runtime.RuntimeIdentity;
|
||||
import org.eclipse.che.workspace.infrastructure.docker.model.DockerEnvironment;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* @author Alexander Garagatyi
|
||||
*/
|
||||
public interface DockerRuntimeFactory {
|
||||
DockerInternalRuntime createRuntime(@Assisted DockerRuntimeContext context,
|
||||
@Assisted String devMachineName,
|
||||
@Assisted List<String> orderedServices,
|
||||
@Assisted DockerEnvironment dockerEnvironment,
|
||||
@Assisted RuntimeIdentity identity);
|
||||
}
|
||||
|
|
@ -12,6 +12,8 @@ package org.eclipse.che.workspace.infrastructure.docker;
|
|||
|
||||
import com.google.inject.Inject;
|
||||
|
||||
import org.eclipse.che.api.agent.server.AgentRegistry;
|
||||
import org.eclipse.che.api.agent.server.impl.AgentSorter;
|
||||
import org.eclipse.che.api.core.ValidationException;
|
||||
import org.eclipse.che.api.core.model.workspace.config.Environment;
|
||||
import org.eclipse.che.api.core.model.workspace.runtime.RuntimeIdentity;
|
||||
|
|
@ -41,7 +43,9 @@ public class DockerRuntimeInfrastructure extends RuntimeInfrastructure {
|
|||
private final ServicesStartStrategy startStrategy;
|
||||
private final InfrastructureProvisioner infrastructureProvisioner;
|
||||
private final EnvironmentNormalizer environmentNormalizer;
|
||||
private final RuntimeFactory runtimeFactory;
|
||||
private final DockerRuntimeFactory runtimeFactory;
|
||||
private final AgentSorter agentSorter;
|
||||
private final AgentRegistry agentRegistry;
|
||||
|
||||
@Inject
|
||||
public DockerRuntimeInfrastructure(EnvironmentParser dockerEnvironmentParser,
|
||||
|
|
@ -50,7 +54,9 @@ public class DockerRuntimeInfrastructure extends RuntimeInfrastructure {
|
|||
InfrastructureProvisioner infrastructureProvisioner,
|
||||
EnvironmentNormalizer environmentNormalizer,
|
||||
Map<String, DockerConfigSourceSpecificEnvironmentParser> environmentParsers,
|
||||
RuntimeFactory runtimeFactory,
|
||||
DockerRuntimeFactory runtimeFactory,
|
||||
AgentSorter agentSorter,
|
||||
AgentRegistry agentRegistry,
|
||||
EventService eventService) {
|
||||
super("docker", environmentParsers.keySet(), eventService);
|
||||
this.dockerEnvironmentValidator = dockerEnvironmentValidator;
|
||||
|
|
@ -59,6 +65,8 @@ public class DockerRuntimeInfrastructure extends RuntimeInfrastructure {
|
|||
this.infrastructureProvisioner = infrastructureProvisioner;
|
||||
this.environmentNormalizer = environmentNormalizer;
|
||||
this.runtimeFactory = runtimeFactory;
|
||||
this.agentSorter = agentSorter;
|
||||
this.agentRegistry = agentRegistry;
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -92,10 +100,13 @@ public class DockerRuntimeInfrastructure extends RuntimeInfrastructure {
|
|||
// normalize env to provide environment description with absolutely everything expected in
|
||||
environmentNormalizer.normalize(environment, dockerEnvironment, identity);
|
||||
|
||||
return runtimeFactory.createContext(this,
|
||||
identity,
|
||||
environment,
|
||||
dockerEnvironment,
|
||||
orderedServices);
|
||||
return new DockerRuntimeContext(this,
|
||||
identity,
|
||||
environment,
|
||||
dockerEnvironment,
|
||||
orderedServices,
|
||||
agentSorter,
|
||||
agentRegistry,
|
||||
runtimeFactory);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -29,6 +29,9 @@ public interface ServerStatusEvent {
|
|||
|
||||
ServerStatusEvent withServerName(String serverName);
|
||||
|
||||
String getServerUrl();
|
||||
|
||||
ServerStatusEvent withServerUrl(String serverUrl);
|
||||
|
||||
String getMachineName();
|
||||
|
||||
|
|
|
|||
|
|
@ -591,6 +591,7 @@ public class WorkspaceManager {
|
|||
try {
|
||||
workspace.setRuntime(runtimes.get(workspace.getId()));
|
||||
} catch (NotFoundException e) {
|
||||
// TODO fix in case of starting ws logs error
|
||||
LOG.error("Workspace " + workspace.getId() + " has RUNNING state but no runtime!");
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -31,9 +31,9 @@ public class MachineImpl implements Machine {
|
|||
return new MachineRuntimeInfoImplBuilder();
|
||||
}
|
||||
|
||||
// public MachineImpl(Machine machineRuntime) {
|
||||
// this(machineRuntime.getProperties(), machineRuntime.getServers());
|
||||
// }
|
||||
public MachineImpl(Machine machineRuntime) {
|
||||
this(machineRuntime.getProperties(), machineRuntime.getServers());
|
||||
}
|
||||
|
||||
public MachineImpl(Map<String, String> properties,
|
||||
Map<String, ? extends Server> servers) {
|
||||
|
|
|
|||
|
|
@ -36,8 +36,16 @@ public class ServerImpl implements Server {
|
|||
return url;
|
||||
}
|
||||
|
||||
public void setUrl(String url) {
|
||||
this.url = url;
|
||||
}
|
||||
|
||||
@Override
|
||||
public ServerStatus getStatus() {
|
||||
return this.status;
|
||||
}
|
||||
|
||||
public void setStatus(ServerStatus status) {
|
||||
this.status = status;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ import org.eclipse.che.api.agent.server.exception.AgentException;
|
|||
import org.eclipse.che.api.agent.server.impl.AgentSorter;
|
||||
import org.eclipse.che.api.agent.shared.model.Agent;
|
||||
import org.eclipse.che.api.agent.shared.model.AgentKey;
|
||||
import org.eclipse.che.api.agent.shared.model.impl.AgentImpl;
|
||||
import org.eclipse.che.api.core.model.workspace.config.MachineConfig;
|
||||
import org.eclipse.che.api.core.model.workspace.config.ServerConfig;
|
||||
|
||||
|
|
@ -23,7 +24,6 @@ import java.util.Collections;
|
|||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
import static java.lang.String.format;
|
||||
|
||||
|
|
@ -35,7 +35,7 @@ import static java.lang.String.format;
|
|||
public class InternalMachineConfig {
|
||||
|
||||
// ordered agent scripts to launch on start
|
||||
private final List<ResolvedAgent> agents;
|
||||
private final List<AgentImpl> agents;
|
||||
// set of servers including ones configured by agents
|
||||
private final Map<String, ServerConfig> servers;
|
||||
private final Map<String, String> attributes;
|
||||
|
|
@ -67,7 +67,7 @@ public class InternalMachineConfig {
|
|||
/**
|
||||
* @return agent scripts
|
||||
*/
|
||||
public List<ResolvedAgent> getAgents() {
|
||||
public List<AgentImpl> getAgents() {
|
||||
return agents;
|
||||
}
|
||||
|
||||
|
|
@ -88,10 +88,7 @@ public class InternalMachineConfig {
|
|||
agentsConf.add(agentRegistry.getAgent(agentKey));
|
||||
}
|
||||
for (Agent agent : agentsConf) {
|
||||
this.agents.add(new ResolvedAgent(agent.getId(),
|
||||
agent.getScript(),
|
||||
agent.getServers().keySet(),
|
||||
agent.getProperties()));
|
||||
this.agents.add(new AgentImpl(agent));
|
||||
for (Map.Entry<String, ? extends ServerConfig> serverEntry : agent.getServers().entrySet()) {
|
||||
if (servers.putIfAbsent(serverEntry.getKey(), serverEntry.getValue()) != null &&
|
||||
servers.get(serverEntry.getKey()).equals(serverEntry.getValue())) {
|
||||
|
|
@ -106,38 +103,4 @@ public class InternalMachineConfig {
|
|||
throw new InfrastructureException(e.getLocalizedMessage(), e);
|
||||
}
|
||||
}
|
||||
|
||||
public static class ResolvedAgent {
|
||||
private String id;
|
||||
private String script;
|
||||
// needed to know which servers should be pinged on start of agent
|
||||
private Set<String> serversRefs;
|
||||
private Map<String, String> properties;
|
||||
|
||||
public ResolvedAgent(String id,
|
||||
String script,
|
||||
Set<String> servers,
|
||||
Map<String, String> properties) {
|
||||
this.id = id;
|
||||
this.script = script;
|
||||
this.serversRefs = servers;
|
||||
this.properties = properties;
|
||||
}
|
||||
|
||||
public String getId() {
|
||||
return id;
|
||||
}
|
||||
|
||||
public String getScript() {
|
||||
return script;
|
||||
}
|
||||
|
||||
public Set<String> getServers() {
|
||||
return serversRefs;
|
||||
}
|
||||
|
||||
public Map<String, String> getProperties() {
|
||||
return properties;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue