Restore snapshots functionality;
Max Shaposhnik
GitHub
parent
fa91db1f4e
commit
8afc4eb053
|
|
@ -47,19 +47,6 @@ public enum WorkspaceStatus {
|
|||
*/
|
||||
RUNNING,
|
||||
|
||||
/**
|
||||
* Workspace is in SNAPSHOTTING status if and only if the workspace
|
||||
* is currently creating snapshots of it's machines.
|
||||
*
|
||||
* <p>Workspace is in SNAPSHOTTING status after it was {@link #RUNNING}.
|
||||
* The status map:
|
||||
* <pre>
|
||||
* RUNNING -> <b>SNAPSHOTTING</b> -> RUNNING (normal behaviour/error while snapshotting)
|
||||
* </pre>
|
||||
* @deprecated move it to Docker env specific
|
||||
*/
|
||||
SNAPSHOTTING,
|
||||
|
||||
/**
|
||||
* Workspace considered as stopping if and only if its active environment is shutting down.
|
||||
*
|
||||
|
|
|
|||
|
|
@ -16,16 +16,21 @@ import org.eclipse.che.api.core.model.workspace.runtime.Machine;
|
|||
import org.eclipse.che.api.core.model.workspace.runtime.Server;
|
||||
import org.eclipse.che.api.workspace.server.spi.InfrastructureException;
|
||||
import org.eclipse.che.api.workspace.server.spi.InternalInfrastructureException;
|
||||
import org.eclipse.che.commons.lang.NameGenerator;
|
||||
import org.eclipse.che.plugin.docker.client.DockerConnector;
|
||||
import org.eclipse.che.plugin.docker.client.Exec;
|
||||
import org.eclipse.che.plugin.docker.client.LogMessage;
|
||||
import org.eclipse.che.plugin.docker.client.MessageProcessor;
|
||||
import org.eclipse.che.plugin.docker.client.ProgressLineFormatterImpl;
|
||||
import org.eclipse.che.plugin.docker.client.json.ContainerInfo;
|
||||
import org.eclipse.che.plugin.docker.client.params.CommitParams;
|
||||
import org.eclipse.che.plugin.docker.client.params.CreateExecParams;
|
||||
import org.eclipse.che.plugin.docker.client.params.PushParams;
|
||||
import org.eclipse.che.plugin.docker.client.params.RemoveContainerParams;
|
||||
import org.eclipse.che.plugin.docker.client.params.RemoveImageParams;
|
||||
import org.eclipse.che.plugin.docker.client.params.StartExecParams;
|
||||
import org.eclipse.che.workspace.infrastructure.docker.monit.DockerMachineStopDetector;
|
||||
import org.eclipse.che.workspace.infrastructure.docker.snapshot.SnapshotException;
|
||||
import org.eclipse.che.workspace.infrastructure.docker.strategy.ServerEvaluationStrategy;
|
||||
import org.eclipse.che.workspace.infrastructure.docker.strategy.ServerEvaluationStrategyProvider;
|
||||
import org.slf4j.Logger;
|
||||
|
|
@ -35,6 +40,8 @@ import java.io.IOException;
|
|||
import java.util.Collections;
|
||||
import java.util.Map;
|
||||
|
||||
import static java.lang.String.format;
|
||||
import static org.eclipse.che.workspace.infrastructure.docker.DockerRegistryClient.MACHINE_SNAPSHOT_PREFIX;
|
||||
import static org.slf4j.LoggerFactory.getLogger;
|
||||
|
||||
/**
|
||||
|
|
@ -83,19 +90,17 @@ public class DockerMachine implements Machine {
|
|||
private final DockerConnector docker;
|
||||
private final String image;
|
||||
private final DockerMachineStopDetector dockerMachineStopDetector;
|
||||
// TODO spi snapshot #5101
|
||||
// private final String registry;
|
||||
// private final String registryNamespace;
|
||||
// private final boolean snapshotUseRegistry;
|
||||
private final String registry;
|
||||
private final String registryNamespace;
|
||||
private final boolean snapshotUseRegistry;
|
||||
private final ContainerInfo info;
|
||||
private final ServerEvaluationStrategyProvider provider;
|
||||
|
||||
@Inject
|
||||
public DockerMachine(DockerConnector docker,
|
||||
// TODO spi snapshot #5101
|
||||
// @Named("che.docker.registry") String registry,
|
||||
// @Named("che.docker.namespace") @Nullable String registryNamespace,
|
||||
// @Named("che.docker.registry_for_snapshots") boolean snapshotUseRegistry,
|
||||
String registry,
|
||||
String registryNamespace,
|
||||
boolean snapshotUseRegistry,
|
||||
@Assisted("container") String container,
|
||||
@Assisted("image") String image,
|
||||
ServerEvaluationStrategyProvider provider,
|
||||
|
|
@ -103,6 +108,9 @@ public class DockerMachine implements Machine {
|
|||
this.container = container;
|
||||
this.docker = docker;
|
||||
this.image = image;
|
||||
this.registry = registry;
|
||||
this.registryNamespace = registryNamespace;
|
||||
this.snapshotUseRegistry = snapshotUseRegistry;
|
||||
this.dockerMachineStopDetector = dockerMachineStopDetector;
|
||||
try {
|
||||
this.info = docker.inspectContainer(container);
|
||||
|
|
@ -163,13 +171,16 @@ public class DockerMachine implements Machine {
|
|||
"container='" + container + '\'' +
|
||||
", docker=" + docker +
|
||||
", image='" + image + '\'' +
|
||||
", registry='" + registry + '\'' +
|
||||
", registryNamespace='" + registryNamespace + '\'' +
|
||||
", snapshotUseRegistry='" + snapshotUseRegistry +
|
||||
", info=" + info +
|
||||
", provider=" + provider +
|
||||
'}';
|
||||
}
|
||||
|
||||
/* TODO spi snapshot #5101
|
||||
public MachineSource saveToSnapshot() throws MachineException {
|
||||
|
||||
public DockerMachineSource saveToSnapshot() throws SnapshotException {
|
||||
try {
|
||||
String image = generateRepository();
|
||||
if(!snapshotUseRegistry) {
|
||||
|
|
@ -188,22 +199,21 @@ public class DockerMachine implements Machine {
|
|||
final ProgressLineFormatterImpl lineFormatter = new ProgressLineFormatterImpl();
|
||||
final String digest = docker.push(pushParams,
|
||||
progressMonitor -> {
|
||||
try {
|
||||
outputConsumer.writeLine(lineFormatter.format(progressMonitor));
|
||||
} catch (IOException ignored) {
|
||||
}
|
||||
// try {
|
||||
// outputConsumer.writeLine(lineFormatter.format(progressMonitor));
|
||||
// } catch (IOException ignored) {
|
||||
// }
|
||||
});
|
||||
docker.removeImage(RemoveImageParams.create(fullRepo).withForce(false));
|
||||
return new DockerMachineSource(image).withRegistry(registry).withDigest(digest).withTag(LATEST_TAG);
|
||||
} catch (IOException ioEx) {
|
||||
throw new MachineException(ioEx);
|
||||
throw new SnapshotException(ioEx);
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
throw new MachineException(e.getLocalizedMessage(), e);
|
||||
throw new SnapshotException(e.getLocalizedMessage(), e);
|
||||
}
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
protected void commitContainer(String repository, String tag) throws IOException {
|
||||
String comment = format("Suspended at %1$ta %1$tb %1$td %1$tT %1$tZ %1$tY",
|
||||
System.currentTimeMillis());
|
||||
|
|
@ -218,8 +228,8 @@ public class DockerMachine implements Machine {
|
|||
|
||||
private String generateRepository() {
|
||||
if (registryNamespace != null) {
|
||||
return registryNamespace + '/' + DockerInstanceProvider.MACHINE_SNAPSHOT_PREFIX + NameGenerator.generate(null, 16);
|
||||
return registryNamespace + '/' + MACHINE_SNAPSHOT_PREFIX + NameGenerator.generate(null, 16);
|
||||
}
|
||||
return DockerInstanceProvider.MACHINE_SNAPSHOT_PREFIX + NameGenerator.generate(null, 16);
|
||||
}*/
|
||||
return MACHINE_SNAPSHOT_PREFIX + NameGenerator.generate(null, 16);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -14,11 +14,13 @@ import com.google.inject.assistedinject.Assisted;
|
|||
|
||||
import org.eclipse.che.api.agent.server.AgentRegistry;
|
||||
import org.eclipse.che.api.agent.server.impl.AgentSorter;
|
||||
import org.eclipse.che.api.core.NotFoundException;
|
||||
import org.eclipse.che.api.core.ValidationException;
|
||||
import org.eclipse.che.api.core.model.machine.MachineSource;
|
||||
import org.eclipse.che.api.core.model.workspace.config.Environment;
|
||||
import org.eclipse.che.api.core.model.workspace.runtime.RuntimeIdentity;
|
||||
import org.eclipse.che.api.workspace.server.URLRewriter;
|
||||
import org.eclipse.che.api.workspace.server.model.impl.MachineSourceImpl;
|
||||
import org.eclipse.che.api.workspace.server.spi.InfrastructureException;
|
||||
import org.eclipse.che.api.workspace.server.spi.InternalInfrastructureException;
|
||||
import org.eclipse.che.api.workspace.server.spi.InternalMachineConfig;
|
||||
|
|
@ -30,11 +32,16 @@ import org.eclipse.che.workspace.infrastructure.docker.exception.SourceNotFoundE
|
|||
import org.eclipse.che.workspace.infrastructure.docker.model.DockerBuildContext;
|
||||
import org.eclipse.che.workspace.infrastructure.docker.model.DockerContainerConfig;
|
||||
import org.eclipse.che.workspace.infrastructure.docker.model.DockerEnvironment;
|
||||
import org.eclipse.che.workspace.infrastructure.docker.snapshot.SnapshotDao;
|
||||
import org.eclipse.che.workspace.infrastructure.docker.snapshot.SnapshotException;
|
||||
import org.eclipse.che.workspace.infrastructure.docker.snapshot.SnapshotImpl;
|
||||
import org.slf4j.Logger;
|
||||
|
||||
import javax.inject.Inject;
|
||||
import java.net.URL;
|
||||
import java.util.ArrayDeque;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
|
|
@ -61,14 +68,16 @@ import static org.slf4j.LoggerFactory.getLogger;
|
|||
public class DockerRuntimeContext extends RuntimeContext {
|
||||
private static final Logger LOG = getLogger(DockerRuntimeContext.class);
|
||||
|
||||
private final NetworkLifecycle dockerNetworkLifecycle;
|
||||
private final MachineStarter serviceStarter;
|
||||
private final DockerEnvironment dockerEnvironment;
|
||||
private final URLRewriter urlRewriter;
|
||||
private final Queue<String> startQueue;
|
||||
private final StartSynchronizer startSynchronizer;
|
||||
private final String devMachineName;
|
||||
private final ContextsStorage contextsStorage;
|
||||
private final NetworkLifecycle dockerNetworkLifecycle;
|
||||
private final MachineStarter serviceStarter;
|
||||
private final DockerEnvironment dockerEnvironment;
|
||||
private final URLRewriter urlRewriter;
|
||||
private final Queue<String> startQueue;
|
||||
private final StartSynchronizer startSynchronizer;
|
||||
private final String devMachineName;
|
||||
private final ContextsStorage contextsStorage;
|
||||
private final SnapshotDao snapshotDao;
|
||||
private final DockerRegistryClient dockerRegistryClient;
|
||||
|
||||
@Inject
|
||||
public DockerRuntimeContext(@Assisted DockerRuntimeInfrastructure infrastructure,
|
||||
|
|
@ -81,7 +90,9 @@ public class DockerRuntimeContext extends RuntimeContext {
|
|||
URLRewriter urlRewriter,
|
||||
AgentSorter agentSorter,
|
||||
AgentRegistry agentRegistry,
|
||||
ContextsStorage contextsStorage)
|
||||
ContextsStorage contextsStorage,
|
||||
SnapshotDao snapshotDao,
|
||||
DockerRegistryClient dockerRegistryClient)
|
||||
throws ValidationException, InfrastructureException {
|
||||
super(environment, identity, infrastructure, agentSorter, agentRegistry);
|
||||
this.devMachineName = Utils.getDevMachineName(environment);
|
||||
|
|
@ -91,6 +102,8 @@ public class DockerRuntimeContext extends RuntimeContext {
|
|||
this.startQueue = new ArrayDeque<>(orderedServices);
|
||||
this.urlRewriter = urlRewriter;
|
||||
this.contextsStorage = contextsStorage;
|
||||
this.snapshotDao = snapshotDao;
|
||||
this.dockerRegistryClient = dockerRegistryClient;
|
||||
this.startSynchronizer = new StartSynchronizer();
|
||||
}
|
||||
|
||||
|
|
@ -121,7 +134,7 @@ public class DockerRuntimeContext extends RuntimeContext {
|
|||
boolean runtimeDestroyingNeeded = !startSynchronizer.isStopCalled();
|
||||
if (runtimeDestroyingNeeded) {
|
||||
try {
|
||||
destroyRuntime();
|
||||
destroyRuntime(null);
|
||||
} catch (Exception destExc) {
|
||||
LOG.error(destExc.getLocalizedMessage(), destExc);
|
||||
}
|
||||
|
|
@ -141,7 +154,7 @@ public class DockerRuntimeContext extends RuntimeContext {
|
|||
protected void internalStop(Map<String, String> stopOptions) throws InfrastructureException {
|
||||
startSynchronizer.interruptStartThread();
|
||||
try {
|
||||
destroyRuntime();
|
||||
destroyRuntime(stopOptions);
|
||||
} finally {
|
||||
contextsStorage.remove(this);
|
||||
}
|
||||
|
|
@ -153,32 +166,43 @@ public class DockerRuntimeContext extends RuntimeContext {
|
|||
}
|
||||
|
||||
private DockerMachine startMachine(String name,
|
||||
DockerContainerConfig container,
|
||||
DockerContainerConfig containerConfig,
|
||||
Map<String, String> startOptions,
|
||||
boolean isDev) throws InfrastructureException {
|
||||
DockerMachine dockerMachine;
|
||||
// TODO property name
|
||||
if ("true".equals(startOptions.get("recover"))) {
|
||||
if ("true".equals(startOptions.get("restore"))) {
|
||||
MachineSourceImpl machineSource = null;
|
||||
try {
|
||||
// TODO set snapshot stuff #5101
|
||||
// container = normalizeSource(container, null);
|
||||
SnapshotImpl snapshot = snapshotDao.getSnapshot(identity.getWorkspaceId(),
|
||||
identity.getEnvName(),
|
||||
name);
|
||||
machineSource = snapshot.getMachineSource();
|
||||
// Snapshot image location has SHA-256 digest which needs to be removed,
|
||||
// otherwise it will be pulled without tag and cause problems
|
||||
String imageName = machineSource.getLocation();
|
||||
if (imageName.contains("@sha256:")) {
|
||||
machineSource.setLocation(imageName.substring(0, imageName.indexOf('@')));
|
||||
}
|
||||
|
||||
DockerContainerConfig imageContainerConfig = normalizeSource(containerConfig, machineSource);
|
||||
dockerMachine = serviceStarter.startService(dockerEnvironment.getNetwork(),
|
||||
name,
|
||||
container,
|
||||
imageContainerConfig,
|
||||
identity,
|
||||
isDev);
|
||||
} catch (SourceNotFoundException e) {
|
||||
} catch (NotFoundException | SnapshotException | SourceNotFoundException e) {
|
||||
// slip to start without recovering
|
||||
dockerMachine = serviceStarter.startService(dockerEnvironment.getNetwork(),
|
||||
name,
|
||||
container,
|
||||
containerConfig,
|
||||
identity,
|
||||
isDev);
|
||||
}
|
||||
} else {
|
||||
dockerMachine = serviceStarter.startService(dockerEnvironment.getNetwork(),
|
||||
name,
|
||||
container,
|
||||
containerConfig,
|
||||
identity,
|
||||
isDev);
|
||||
}
|
||||
|
|
@ -192,24 +216,26 @@ public class DockerRuntimeContext extends RuntimeContext {
|
|||
startSynchronizer.getMachines());
|
||||
}
|
||||
|
||||
private void destroyRuntime() throws InfrastructureException {
|
||||
for (Map.Entry<String, DockerMachine> dockerMachineEntry : startSynchronizer.removeMachines().entrySet()) {
|
||||
try {
|
||||
// TODO spi snapshot #5101
|
||||
dockerMachineEntry.getValue().destroy();
|
||||
} catch (InfrastructureException e) {
|
||||
LOG.error(format("Error occurs on destroying of docker machine '%s' in workspace '%s'. Container '%s'",
|
||||
dockerMachineEntry.getKey(),
|
||||
getIdentity().getWorkspaceId(),
|
||||
dockerMachineEntry.getValue().getContainer()),
|
||||
e);
|
||||
private void destroyRuntime(Map<String, String> stopOptions) throws InfrastructureException {
|
||||
if (stopOptions != null && "true".equals(stopOptions.get("create-snapshot"))) {
|
||||
snapshotMachines(startSynchronizer.removeMachines());
|
||||
} else {
|
||||
for (Map.Entry<String, DockerMachine> dockerMachineEntry : startSynchronizer.removeMachines().entrySet()) {
|
||||
try {
|
||||
dockerMachineEntry.getValue().destroy();
|
||||
} catch (InfrastructureException e) {
|
||||
LOG.error(format("Error occurs on destroying of docker machine '%s' in workspace '%s'. Container '%s'",
|
||||
dockerMachineEntry.getKey(),
|
||||
getIdentity().getWorkspaceId(),
|
||||
dockerMachineEntry.getValue().getContainer()),
|
||||
e);
|
||||
}
|
||||
}
|
||||
}
|
||||
// TODO what happens when context throws exception here
|
||||
dockerNetworkLifecycle.destroyNetwork(dockerEnvironment.getNetwork());
|
||||
}
|
||||
|
||||
// TODO Do not remove, will be used in snapshot #5101
|
||||
private DockerContainerConfig normalizeSource(DockerContainerConfig containerConfig,
|
||||
MachineSource machineSource) {
|
||||
DockerContainerConfig serviceWithNormalizedSource = new DockerContainerConfig(containerConfig);
|
||||
|
|
@ -234,6 +260,68 @@ public class DockerRuntimeContext extends RuntimeContext {
|
|||
return serviceWithNormalizedSource;
|
||||
}
|
||||
|
||||
/**
|
||||
* Removes binaries of all the snapshots, continues to remove
|
||||
* snapshots if removal of binaries for a single snapshot fails.
|
||||
*
|
||||
* @param snapshots
|
||||
* the list of snapshots to remove binaries
|
||||
*/
|
||||
private void removeBinaries(Collection<? extends SnapshotImpl> snapshots) {
|
||||
for (SnapshotImpl snapshot : snapshots) {
|
||||
try {
|
||||
dockerRegistryClient.removeInstanceSnapshot(snapshot.getMachineSource());
|
||||
} catch (SnapshotException x) {
|
||||
LOG.error(format("Couldn't remove snapshot '%s', workspace id '%s'", snapshot.getId(), snapshot.getWorkspaceId()), x);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Prepare snapshots of all active machines.
|
||||
* @param machines
|
||||
* the active machines map
|
||||
*/
|
||||
private void snapshotMachines(Map<String, DockerMachine> machines) {
|
||||
List<SnapshotImpl> newSnapshots = new ArrayList<>();
|
||||
for (Map.Entry<String, DockerMachine> dockerMachineEntry : machines.entrySet()) {
|
||||
SnapshotImpl snapshot = SnapshotImpl.builder()
|
||||
.generateId()
|
||||
.setType("docker") //TODO: do we need that at all?
|
||||
.setWorkspaceId(identity.getWorkspaceId())
|
||||
.setDescription(identity.getEnvName())
|
||||
.setDev(dockerMachineEntry.getKey().equals(devMachineName))
|
||||
.setEnvName(identity.getEnvName())
|
||||
.setMachineName(dockerMachineEntry.getKey())
|
||||
.useCurrentCreationDate()
|
||||
.build();
|
||||
try {
|
||||
DockerMachineSource machineSource = dockerMachineEntry.getValue().saveToSnapshot();
|
||||
snapshot.setMachineSource(new MachineSourceImpl(machineSource));
|
||||
newSnapshots.add(snapshot);
|
||||
} catch (SnapshotException e) {
|
||||
LOG.error(format("Error occurs on snapshotting of docker machine '%s' in workspace '%s'. Container '%s'",
|
||||
dockerMachineEntry.getKey(),
|
||||
getIdentity().getWorkspaceId(),
|
||||
dockerMachineEntry.getValue().getContainer()),
|
||||
e);
|
||||
}
|
||||
}
|
||||
try {
|
||||
List<SnapshotImpl> removed = snapshotDao.replaceSnapshots(identity.getWorkspaceId(),
|
||||
identity.getEnvName(),
|
||||
newSnapshots);
|
||||
if (!removed.isEmpty()) {
|
||||
LOG.info("Removing old snapshots binaries, workspace id '{}', snapshots to remove '{}'", identity.getWorkspaceId(),
|
||||
removed.size());
|
||||
removeBinaries(removed);
|
||||
}
|
||||
} catch (SnapshotException e) {
|
||||
LOG.error(format("Couldn't remove existing snapshots metadata for workspace '%s'", identity.getWorkspaceId()), e);
|
||||
removeBinaries(newSnapshots);
|
||||
}
|
||||
}
|
||||
|
||||
// TODO rework to agent launchers
|
||||
private void startAgents(String machineName, DockerMachine dockerMachine) throws InfrastructureException {
|
||||
InternalMachineConfig machineConfig = internalMachines.get(machineName);
|
||||
|
|
|
|||
|
|
@ -135,6 +135,8 @@ public class MachineStarter {
|
|||
private final Set<String> additionalNetworks;
|
||||
private final String parentCgroup;
|
||||
private final String cpusetCpus;
|
||||
private final String registry;
|
||||
private final String registryNamespace;
|
||||
private final long cpuPeriod;
|
||||
private final long cpuQuota;
|
||||
private final WindowsPathEscaper windowsPathEscaper;
|
||||
|
|
@ -153,6 +155,8 @@ public class MachineStarter {
|
|||
@Named("che.docker.always_pull_image") boolean doForcePullImage,
|
||||
@Named("che.docker.privileged") boolean privilegedMode,
|
||||
@Named("che.docker.pids_limit") int pidsLimit,
|
||||
@Named("che.docker.registry") String registry,
|
||||
@Named("che.docker.namespace") @Nullable String registryNamespace,
|
||||
@Named("machine.docker.dev_machine.machine_env") Set<String> devMachineEnvVariables,
|
||||
@Named("machine.docker.machine_env") Set<String> allMachinesEnvVariables,
|
||||
@Named("che.docker.registry_for_snapshots") boolean snapshotUseRegistry,
|
||||
|
|
@ -188,6 +192,8 @@ public class MachineStarter {
|
|||
this.cpuPeriod = cpuPeriod;
|
||||
this.cpuQuota = cpuQuota;
|
||||
this.windowsPathEscaper = windowsPathEscaper;
|
||||
this.registryNamespace = registryNamespace;
|
||||
this.registry = registry;
|
||||
this.pidsLimit = pidsLimit;
|
||||
this.dnsResolvers = dnsResolvers;
|
||||
this.buildArgs = buildArgs;
|
||||
|
|
@ -335,6 +341,9 @@ public class MachineStarter {
|
|||
workspaceId);
|
||||
|
||||
return new DockerMachine(docker,
|
||||
registry,
|
||||
registryNamespace,
|
||||
snapshotUseRegistry,
|
||||
container,
|
||||
image,
|
||||
serverEvaluationStrategyProvider,
|
||||
|
|
|
|||
Loading…
Reference in New Issue