diff --git a/core/che-core-api-model/src/main/java/org/eclipse/che/api/core/model/workspace/WorkspaceStatus.java b/core/che-core-api-model/src/main/java/org/eclipse/che/api/core/model/workspace/WorkspaceStatus.java index 8924eec4ac..35d0f97d41 100644 --- a/core/che-core-api-model/src/main/java/org/eclipse/che/api/core/model/workspace/WorkspaceStatus.java +++ b/core/che-core-api-model/src/main/java/org/eclipse/che/api/core/model/workspace/WorkspaceStatus.java @@ -47,19 +47,6 @@ public enum WorkspaceStatus { */ RUNNING, - /** - * Workspace is in SNAPSHOTTING status if and only if the workspace - * is currently creating snapshots of it's machines. - * - *

Workspace is in SNAPSHOTTING status after it was {@link #RUNNING}. - * The status map: - * 

-     *     RUNNING -> SNAPSHOTTING -> RUNNING (normal behaviour/error while snapshotting)
-     *
- * @deprecated move it to Docker env specific - */ - SNAPSHOTTING, - /** * Workspace considered as stopping if and only if its active environment is shutting down. * diff --git a/infrastructures/docker/src/main/java/org/eclipse/che/workspace/infrastructure/docker/DockerMachine.java b/infrastructures/docker/src/main/java/org/eclipse/che/workspace/infrastructure/docker/DockerMachine.java index 3094f0fac7..18b6fcd89c 100644 --- a/infrastructures/docker/src/main/java/org/eclipse/che/workspace/infrastructure/docker/DockerMachine.java +++ b/infrastructures/docker/src/main/java/org/eclipse/che/workspace/infrastructure/docker/DockerMachine.java @@ -16,16 +16,21 @@ import org.eclipse.che.api.core.model.workspace.runtime.Machine; import org.eclipse.che.api.core.model.workspace.runtime.Server; import org.eclipse.che.api.workspace.server.spi.InfrastructureException; import org.eclipse.che.api.workspace.server.spi.InternalInfrastructureException; +import org.eclipse.che.commons.lang.NameGenerator; import org.eclipse.che.plugin.docker.client.DockerConnector; import org.eclipse.che.plugin.docker.client.Exec; import org.eclipse.che.plugin.docker.client.LogMessage; import org.eclipse.che.plugin.docker.client.MessageProcessor; +import org.eclipse.che.plugin.docker.client.ProgressLineFormatterImpl; import org.eclipse.che.plugin.docker.client.json.ContainerInfo; +import org.eclipse.che.plugin.docker.client.params.CommitParams; import org.eclipse.che.plugin.docker.client.params.CreateExecParams; +import org.eclipse.che.plugin.docker.client.params.PushParams; import org.eclipse.che.plugin.docker.client.params.RemoveContainerParams; import org.eclipse.che.plugin.docker.client.params.RemoveImageParams; import org.eclipse.che.plugin.docker.client.params.StartExecParams; import org.eclipse.che.workspace.infrastructure.docker.monit.DockerMachineStopDetector; +import org.eclipse.che.workspace.infrastructure.docker.snapshot.SnapshotException; import org.eclipse.che.workspace.infrastructure.docker.strategy.ServerEvaluationStrategy; import org.eclipse.che.workspace.infrastructure.docker.strategy.ServerEvaluationStrategyProvider; import org.slf4j.Logger; @@ -35,6 +40,8 @@ import java.io.IOException; import java.util.Collections; import java.util.Map; +import static java.lang.String.format; +import static org.eclipse.che.workspace.infrastructure.docker.DockerRegistryClient.MACHINE_SNAPSHOT_PREFIX; import static org.slf4j.LoggerFactory.getLogger; /** @@ -83,19 +90,17 @@ public class DockerMachine implements Machine { private final DockerConnector docker; private final String image; private final DockerMachineStopDetector dockerMachineStopDetector; - // TODO spi snapshot #5101 -// private final String registry; -// private final String registryNamespace; -// private final boolean snapshotUseRegistry; + private final String registry; + private final String registryNamespace; + private final boolean snapshotUseRegistry; private final ContainerInfo info; private final ServerEvaluationStrategyProvider provider; @Inject public DockerMachine(DockerConnector docker, - // TODO spi snapshot #5101 -// @Named("che.docker.registry") String registry, -// @Named("che.docker.namespace") @Nullable String registryNamespace, -// @Named("che.docker.registry_for_snapshots") boolean snapshotUseRegistry, + String registry, + String registryNamespace, + boolean snapshotUseRegistry, @Assisted("container") String container, @Assisted("image") String image, ServerEvaluationStrategyProvider provider, @@ -103,6 +108,9 @@ public class DockerMachine implements Machine { this.container = container; this.docker = docker; this.image = image; + this.registry = registry; + this.registryNamespace = registryNamespace; + this.snapshotUseRegistry = snapshotUseRegistry; this.dockerMachineStopDetector = dockerMachineStopDetector; try { this.info = docker.inspectContainer(container); @@ -163,13 +171,16 @@ public class DockerMachine implements Machine { "container='" + container + '\'' + ", docker=" + docker + ", image='" + image + '\'' + + ", registry='" + registry + '\'' + + ", registryNamespace='" + registryNamespace + '\'' + + ", snapshotUseRegistry='" + snapshotUseRegistry + ", info=" + info + ", provider=" + provider + '}'; } - /* TODO spi snapshot #5101 - public MachineSource saveToSnapshot() throws MachineException { + + public DockerMachineSource saveToSnapshot() throws SnapshotException { try { String image = generateRepository(); if(!snapshotUseRegistry) { @@ -188,22 +199,21 @@ public class DockerMachine implements Machine { final ProgressLineFormatterImpl lineFormatter = new ProgressLineFormatterImpl(); final String digest = docker.push(pushParams, progressMonitor -> { - try { - outputConsumer.writeLine(lineFormatter.format(progressMonitor)); - } catch (IOException ignored) { - } +// try { +// outputConsumer.writeLine(lineFormatter.format(progressMonitor)); +// } catch (IOException ignored) { +// } }); docker.removeImage(RemoveImageParams.create(fullRepo).withForce(false)); return new DockerMachineSource(image).withRegistry(registry).withDigest(digest).withTag(LATEST_TAG); } catch (IOException ioEx) { - throw new MachineException(ioEx); + throw new SnapshotException(ioEx); } catch (InterruptedException e) { Thread.currentThread().interrupt(); - throw new MachineException(e.getLocalizedMessage(), e); + throw new SnapshotException(e.getLocalizedMessage(), e); } } - @VisibleForTesting protected void commitContainer(String repository, String tag) throws IOException { String comment = format("Suspended at %1$ta %1$tb %1$td %1$tT %1$tZ %1$tY", System.currentTimeMillis()); @@ -218,8 +228,8 @@ public class DockerMachine implements Machine { private String generateRepository() { if (registryNamespace != null) { - return registryNamespace + '/' + DockerInstanceProvider.MACHINE_SNAPSHOT_PREFIX + NameGenerator.generate(null, 16); + return registryNamespace + '/' + MACHINE_SNAPSHOT_PREFIX + NameGenerator.generate(null, 16); } - return DockerInstanceProvider.MACHINE_SNAPSHOT_PREFIX + NameGenerator.generate(null, 16); - }*/ + return MACHINE_SNAPSHOT_PREFIX + NameGenerator.generate(null, 16); + } } diff --git a/infrastructures/docker/src/main/java/org/eclipse/che/workspace/infrastructure/docker/DockerRuntimeContext.java b/infrastructures/docker/src/main/java/org/eclipse/che/workspace/infrastructure/docker/DockerRuntimeContext.java index 46166e05c7..e3bd065ef7 100644 --- a/infrastructures/docker/src/main/java/org/eclipse/che/workspace/infrastructure/docker/DockerRuntimeContext.java +++ b/infrastructures/docker/src/main/java/org/eclipse/che/workspace/infrastructure/docker/DockerRuntimeContext.java @@ -14,11 +14,13 @@ import com.google.inject.assistedinject.Assisted; import org.eclipse.che.api.agent.server.AgentRegistry; import org.eclipse.che.api.agent.server.impl.AgentSorter; +import org.eclipse.che.api.core.NotFoundException; import org.eclipse.che.api.core.ValidationException; import org.eclipse.che.api.core.model.machine.MachineSource; import org.eclipse.che.api.core.model.workspace.config.Environment; import org.eclipse.che.api.core.model.workspace.runtime.RuntimeIdentity; import org.eclipse.che.api.workspace.server.URLRewriter; +import org.eclipse.che.api.workspace.server.model.impl.MachineSourceImpl; import org.eclipse.che.api.workspace.server.spi.InfrastructureException; import org.eclipse.che.api.workspace.server.spi.InternalInfrastructureException; import org.eclipse.che.api.workspace.server.spi.InternalMachineConfig; @@ -30,11 +32,16 @@ import org.eclipse.che.workspace.infrastructure.docker.exception.SourceNotFoundE import org.eclipse.che.workspace.infrastructure.docker.model.DockerBuildContext; import org.eclipse.che.workspace.infrastructure.docker.model.DockerContainerConfig; import org.eclipse.che.workspace.infrastructure.docker.model.DockerEnvironment; +import org.eclipse.che.workspace.infrastructure.docker.snapshot.SnapshotDao; +import org.eclipse.che.workspace.infrastructure.docker.snapshot.SnapshotException; +import org.eclipse.che.workspace.infrastructure.docker.snapshot.SnapshotImpl; import org.slf4j.Logger; import javax.inject.Inject; import java.net.URL; import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -61,14 +68,16 @@ import static org.slf4j.LoggerFactory.getLogger; public class DockerRuntimeContext extends RuntimeContext { private static final Logger LOG = getLogger(DockerRuntimeContext.class); - private final NetworkLifecycle dockerNetworkLifecycle; - private final MachineStarter serviceStarter; - private final DockerEnvironment dockerEnvironment; - private final URLRewriter urlRewriter; - private final Queue startQueue; - private final StartSynchronizer startSynchronizer; - private final String devMachineName; - private final ContextsStorage contextsStorage; + private final NetworkLifecycle dockerNetworkLifecycle; + private final MachineStarter serviceStarter; + private final DockerEnvironment dockerEnvironment; + private final URLRewriter urlRewriter; + private final Queue startQueue; + private final StartSynchronizer startSynchronizer; + private final String devMachineName; + private final ContextsStorage contextsStorage; + private final SnapshotDao snapshotDao; + private final DockerRegistryClient dockerRegistryClient; @Inject public DockerRuntimeContext(@Assisted DockerRuntimeInfrastructure infrastructure, @@ -81,7 +90,9 @@ public class DockerRuntimeContext extends RuntimeContext { URLRewriter urlRewriter, AgentSorter agentSorter, AgentRegistry agentRegistry, - ContextsStorage contextsStorage) + ContextsStorage contextsStorage, + SnapshotDao snapshotDao, + DockerRegistryClient dockerRegistryClient) throws ValidationException, InfrastructureException { super(environment, identity, infrastructure, agentSorter, agentRegistry); this.devMachineName = Utils.getDevMachineName(environment); @@ -91,6 +102,8 @@ public class DockerRuntimeContext extends RuntimeContext { this.startQueue = new ArrayDeque<>(orderedServices); this.urlRewriter = urlRewriter; this.contextsStorage = contextsStorage; + this.snapshotDao = snapshotDao; + this.dockerRegistryClient = dockerRegistryClient; this.startSynchronizer = new StartSynchronizer(); } @@ -121,7 +134,7 @@ public class DockerRuntimeContext extends RuntimeContext { boolean runtimeDestroyingNeeded = !startSynchronizer.isStopCalled(); if (runtimeDestroyingNeeded) { try { - destroyRuntime(); + destroyRuntime(null); } catch (Exception destExc) { LOG.error(destExc.getLocalizedMessage(), destExc); } @@ -141,7 +154,7 @@ public class DockerRuntimeContext extends RuntimeContext { protected void internalStop(Map stopOptions) throws InfrastructureException { startSynchronizer.interruptStartThread(); try { - destroyRuntime(); + destroyRuntime(stopOptions); } finally { contextsStorage.remove(this); } @@ -153,32 +166,43 @@ public class DockerRuntimeContext extends RuntimeContext { } private DockerMachine startMachine(String name, - DockerContainerConfig container, + DockerContainerConfig containerConfig, Map startOptions, boolean isDev) throws InfrastructureException { DockerMachine dockerMachine; // TODO property name - if ("true".equals(startOptions.get("recover"))) { + if ("true".equals(startOptions.get("restore"))) { + MachineSourceImpl machineSource = null; try { - // TODO set snapshot stuff #5101 -// container = normalizeSource(container, null); + SnapshotImpl snapshot = snapshotDao.getSnapshot(identity.getWorkspaceId(), + identity.getEnvName(), + name); + machineSource = snapshot.getMachineSource(); + // Snapshot image location has SHA-256 digest which needs to be removed, + // otherwise it will be pulled without tag and cause problems + String imageName = machineSource.getLocation(); + if (imageName.contains("@sha256:")) { + machineSource.setLocation(imageName.substring(0, imageName.indexOf('@'))); + } + + DockerContainerConfig imageContainerConfig = normalizeSource(containerConfig, machineSource); dockerMachine = serviceStarter.startService(dockerEnvironment.getNetwork(), name, - container, + imageContainerConfig, identity, isDev); - } catch (SourceNotFoundException e) { + } catch (NotFoundException | SnapshotException | SourceNotFoundException e) { // slip to start without recovering dockerMachine = serviceStarter.startService(dockerEnvironment.getNetwork(), name, - container, + containerConfig, identity, isDev); } } else { dockerMachine = serviceStarter.startService(dockerEnvironment.getNetwork(), name, - container, + containerConfig, identity, isDev); } @@ -192,24 +216,26 @@ public class DockerRuntimeContext extends RuntimeContext { startSynchronizer.getMachines()); } - private void destroyRuntime() throws InfrastructureException { - for (Map.Entry dockerMachineEntry : startSynchronizer.removeMachines().entrySet()) { - try { - // TODO spi snapshot #5101 - dockerMachineEntry.getValue().destroy(); - } catch (InfrastructureException e) { - LOG.error(format("Error occurs on destroying of docker machine '%s' in workspace '%s'. Container '%s'", - dockerMachineEntry.getKey(), - getIdentity().getWorkspaceId(), - dockerMachineEntry.getValue().getContainer()), - e); + private void destroyRuntime(Map stopOptions) throws InfrastructureException { + if (stopOptions != null && "true".equals(stopOptions.get("create-snapshot"))) { + snapshotMachines(startSynchronizer.removeMachines()); + } else { + for (Map.Entry dockerMachineEntry : startSynchronizer.removeMachines().entrySet()) { + try { + dockerMachineEntry.getValue().destroy(); + } catch (InfrastructureException e) { + LOG.error(format("Error occurs on destroying of docker machine '%s' in workspace '%s'. Container '%s'", + dockerMachineEntry.getKey(), + getIdentity().getWorkspaceId(), + dockerMachineEntry.getValue().getContainer()), + e); + } } } // TODO what happens when context throws exception here dockerNetworkLifecycle.destroyNetwork(dockerEnvironment.getNetwork()); } - // TODO Do not remove, will be used in snapshot #5101 private DockerContainerConfig normalizeSource(DockerContainerConfig containerConfig, MachineSource machineSource) { DockerContainerConfig serviceWithNormalizedSource = new DockerContainerConfig(containerConfig); @@ -234,6 +260,68 @@ public class DockerRuntimeContext extends RuntimeContext { return serviceWithNormalizedSource; } + /** + * Removes binaries of all the snapshots, continues to remove + * snapshots if removal of binaries for a single snapshot fails. + * + * @param snapshots + * the list of snapshots to remove binaries + */ + private void removeBinaries(Collection snapshots) { + for (SnapshotImpl snapshot : snapshots) { + try { + dockerRegistryClient.removeInstanceSnapshot(snapshot.getMachineSource()); + } catch (SnapshotException x) { + LOG.error(format("Couldn't remove snapshot '%s', workspace id '%s'", snapshot.getId(), snapshot.getWorkspaceId()), x); + } + } + } + + /** + * Prepare snapshots of all active machines. + * @param machines + * the active machines map + */ + private void snapshotMachines(Map machines) { + List newSnapshots = new ArrayList<>(); + for (Map.Entry dockerMachineEntry : machines.entrySet()) { + SnapshotImpl snapshot = SnapshotImpl.builder() + .generateId() + .setType("docker") //TODO: do we need that at all? + .setWorkspaceId(identity.getWorkspaceId()) + .setDescription(identity.getEnvName()) + .setDev(dockerMachineEntry.getKey().equals(devMachineName)) + .setEnvName(identity.getEnvName()) + .setMachineName(dockerMachineEntry.getKey()) + .useCurrentCreationDate() + .build(); + try { + DockerMachineSource machineSource = dockerMachineEntry.getValue().saveToSnapshot(); + snapshot.setMachineSource(new MachineSourceImpl(machineSource)); + newSnapshots.add(snapshot); + } catch (SnapshotException e) { + LOG.error(format("Error occurs on snapshotting of docker machine '%s' in workspace '%s'. Container '%s'", + dockerMachineEntry.getKey(), + getIdentity().getWorkspaceId(), + dockerMachineEntry.getValue().getContainer()), + e); + } + } + try { + List removed = snapshotDao.replaceSnapshots(identity.getWorkspaceId(), + identity.getEnvName(), + newSnapshots); + if (!removed.isEmpty()) { + LOG.info("Removing old snapshots binaries, workspace id '{}', snapshots to remove '{}'", identity.getWorkspaceId(), + removed.size()); + removeBinaries(removed); + } + } catch (SnapshotException e) { + LOG.error(format("Couldn't remove existing snapshots metadata for workspace '%s'", identity.getWorkspaceId()), e); + removeBinaries(newSnapshots); + } + } + // TODO rework to agent launchers private void startAgents(String machineName, DockerMachine dockerMachine) throws InfrastructureException { InternalMachineConfig machineConfig = internalMachines.get(machineName); diff --git a/infrastructures/docker/src/main/java/org/eclipse/che/workspace/infrastructure/docker/MachineStarter.java b/infrastructures/docker/src/main/java/org/eclipse/che/workspace/infrastructure/docker/MachineStarter.java index 8fae2bfde8..9e052476e0 100644 --- a/infrastructures/docker/src/main/java/org/eclipse/che/workspace/infrastructure/docker/MachineStarter.java +++ b/infrastructures/docker/src/main/java/org/eclipse/che/workspace/infrastructure/docker/MachineStarter.java @@ -135,6 +135,8 @@ public class MachineStarter { private final Set additionalNetworks; private final String parentCgroup; private final String cpusetCpus; + private final String registry; + private final String registryNamespace; private final long cpuPeriod; private final long cpuQuota; private final WindowsPathEscaper windowsPathEscaper; @@ -153,6 +155,8 @@ public class MachineStarter { @Named("che.docker.always_pull_image") boolean doForcePullImage, @Named("che.docker.privileged") boolean privilegedMode, @Named("che.docker.pids_limit") int pidsLimit, + @Named("che.docker.registry") String registry, + @Named("che.docker.namespace") @Nullable String registryNamespace, @Named("machine.docker.dev_machine.machine_env") Set devMachineEnvVariables, @Named("machine.docker.machine_env") Set allMachinesEnvVariables, @Named("che.docker.registry_for_snapshots") boolean snapshotUseRegistry, @@ -188,6 +192,8 @@ public class MachineStarter { this.cpuPeriod = cpuPeriod; this.cpuQuota = cpuQuota; this.windowsPathEscaper = windowsPathEscaper; + this.registryNamespace = registryNamespace; + this.registry = registry; this.pidsLimit = pidsLimit; this.dnsResolvers = dnsResolvers; this.buildArgs = buildArgs; @@ -335,6 +341,9 @@ public class MachineStarter { workspaceId); return new DockerMachine(docker, + registry, + registryNamespace, + snapshotUseRegistry, container, image, serverEvaluationStrategyProvider,