seg_yinzy
/
che-server
mirror of https://github.com/eclipse-che/che-server.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Remove che.infra.kubernetes.async.storage.image, che.infra.kubernetes.async.storage.shutdown_timeout_min, che.infra.kubernetes.async.storage.shutdown_check_period_min properties and some PVC-related classes 

Signed-off-by: Andrew Obuchowicz <aobuchow@redhat.com>
pull/344/head
Andrew Obuchowicz 2022-08-02 12:25:54 -04:00 committed by Ilya Buziuk
parent f107c5066c
commit d9870829fa
20 changed files with 1 additions and 2388 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
assembly/assembly-wsmaster-war/src/main/webapp/WEB-INF/classes/che/che.properties
View File

@ -690,9 +690,6 @@ che.workspace.provision.secret.labels=app.kubernetes.io/part-of=che.eclipse.org,
# and supported by environment
che.workspace.devfile.async.storage.plugin=eclipse/che-async-pv-plugin/latest
# Docker image for the {prod-short} asynchronous storage
che.infra.kubernetes.async.storage.image=quay.io/eclipse/che-workspace-data-sync-storage:0.0.1
# Optionally configures node selector for workspace Pod. Format is comma-separated
# key=value pairs, for example: `disktype=ssd,cpu=xlarge,foo=bar`
che.workspace.pod.node_selector=NULL
@ -703,13 +700,6 @@ che.workspace.pod.node_selector=NULL
# Example: `[{"effect":"NoExecute","key":"aNodeTaint","operator":"Equal","value":"aValue"}]`
che.workspace.pod.tolerations_json=NULL
# The timeout for the Asynchronous Storage Pod shutdown after stopping the last used workspace.
# Value less or equal to 0 interpreted as disabling shutdown ability.
che.infra.kubernetes.async.storage.shutdown_timeout_min=120
# Defines the period with which the Asynchronous Storage Pod stopping ability will be performed (once in 30 minutes by default)
che.infra.kubernetes.async.storage.shutdown_check_period_min=30
# Bitbucket endpoints used for factory integrations.
# Comma separated list of Bitbucket server URLs or NULL if no integration expected.
che.integration.bitbucket.server_endpoints=NULL

144
infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/AsyncStorageModeValidator.java
View File

@ -1,144 +0,0 @@
/*
* Copyright (c) 2012-2022 Red Hat, Inc.
* This program and the accompanying materials are made
* available under the terms of the Eclipse Public License 2.0
* which is available at https://www.eclipse.org/legal/epl-2.0/
*
* SPDX-License-Identifier: EPL-2.0
*
* Contributors:
* Red Hat, Inc. - initial API and implementation
*/
package org.eclipse.che.workspace.infrastructure.kubernetes;
import static com.google.common.base.Strings.isNullOrEmpty;
import static java.lang.Boolean.parseBoolean;
import static java.lang.String.format;
import static org.eclipse.che.api.workspace.shared.Constants.ASYNC_PERSIST_ATTRIBUTE;
import static org.eclipse.che.workspace.infrastructure.kubernetes.namespace.pvc.CommonPVCStrategy.COMMON_STRATEGY;
import static org.eclipse.che.workspace.infrastructure.kubernetes.namespace.pvc.EphemeralWorkspaceUtility.isEphemeral;
import java.util.Map;
import javax.inject.Inject;
import javax.inject.Named;
import org.eclipse.che.api.core.ValidationException;
import org.eclipse.che.api.workspace.server.WorkspaceAttributeValidator;
import org.eclipse.che.commons.annotation.Nullable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Validates the workspace attributes before workspace creation and updating if async storage
* configure.
*
* <p>To be valid for async storage workspace MUST have attributes:
*
* <ul>
* <li>{@link org.eclipse.che.api.workspace.shared.Constants#ASYNC_PERSIST_ATTRIBUTE} = 'true'
* <li>{@link org.eclipse.che.api.workspace.shared.Constants#PERSIST_VOLUMES_ATTRIBUTE} = 'false'
* </ul>
*
* <p>If set only {@link org.eclipse.che.api.workspace.shared.Constants#ASYNC_PERSIST_ATTRIBUTE} =
* 'true', {@link ValidationException} is thrown.
*
* <p>If system is configured with other value of properties than below {@link ValidationException},
* is thrown.
*
* <ul>
* <li>che.infra.kubernetes.namespace.default=<username>-che
* <li>che.infra.kubernetes.pvc.strategy=common
* <li>che.limits.user.workspaces.run.count=1
* </ul>
*/
public class AsyncStorageModeValidator implements WorkspaceAttributeValidator {
private static final Logger LOG = LoggerFactory.getLogger(AsyncStorageModeValidator.class);
private final String pvcStrategy;
private final int runtimesPerUser;
private final boolean isNamespaceStrategyNotValid;
private final boolean isPvcStrategyNotValid;
private final boolean singleRuntimeAllowed;
@Inject
public AsyncStorageModeValidator(
@Nullable @Named("che.infra.kubernetes.namespace.default") String defaultNamespaceName,
@Named("che.limits.user.workspaces.run.count") int runtimesPerUser) {
this.pvcStrategy = "TEST";
this.runtimesPerUser = runtimesPerUser;
this.isPvcStrategyNotValid = !COMMON_STRATEGY.equals(pvcStrategy);
this.singleRuntimeAllowed = runtimesPerUser == 1;
this.isNamespaceStrategyNotValid =
isNullOrEmpty(defaultNamespaceName) || !defaultNamespaceName.contains("<username>");
}
@Override
public void validate(Map<String, String> attributes) throws ValidationException {
if (parseBoolean(attributes.get(ASYNC_PERSIST_ATTRIBUTE))) {
isEphemeralAttributeValidation(attributes);
pvcStrategyValidation();
nameSpaceStrategyValidation();
runtimesPerUserValidation();
}
}
@Override
public void validateUpdate(Map<String, String> existing, Map<String, String> update)
throws ValidationException {
if (parseBoolean(update.get(ASYNC_PERSIST_ATTRIBUTE))) {
if (isEphemeral(existing) || isEphemeral(update)) {
pvcStrategyValidation();
nameSpaceStrategyValidation();
runtimesPerUserValidation();
} else {
String message =
"Workspace configuration not valid: Asynchronous storage available only for NOT persistent storage";
LOG.warn(message);
throw new ValidationException(message);
}
}
}
private void isEphemeralAttributeValidation(Map<String, String> attributes)
throws ValidationException {
if (!isEphemeral(attributes)) {
String message =
"Workspace configuration not valid: Asynchronous storage available only for NOT persistent storage";
LOG.warn(message);
throw new ValidationException(message);
}
}
private void runtimesPerUserValidation() throws ValidationException {
if (!singleRuntimeAllowed) {
String message =
format(
"Workspace configuration not valid: Asynchronous storage available only if 'che.limits.user.workspaces.run.count' set to 1, but got %s",
runtimesPerUser);
LOG.warn(message);
throw new ValidationException(message);
}
}
private void nameSpaceStrategyValidation() throws ValidationException {
if (isNamespaceStrategyNotValid) {
String message =
"Workspace configuration not valid: Asynchronous storage available only for 'per-user' namespace strategy";
LOG.warn(message);
throw new ValidationException(message);
}
}
private void pvcStrategyValidation() throws ValidationException {
if (isPvcStrategyNotValid) {
String message =
format(
"Workspace configuration not valid: Asynchronous storage available only for 'common' PVC strategy, but got %s",
pvcStrategy);
LOG.warn(message);
throw new ValidationException(message);
}
}
}

13
infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesEnvironmentProvisioner.java
View File

@ -18,14 +18,11 @@ import org.eclipse.che.api.workspace.server.spi.InfrastructureException;
import org.eclipse.che.commons.annotation.Traced;
import org.eclipse.che.commons.tracing.TracingTags;
import org.eclipse.che.workspace.infrastructure.kubernetes.environment.KubernetesEnvironment;
import org.eclipse.che.workspace.infrastructure.kubernetes.provision.AsyncStoragePodInterceptor;
import org.eclipse.che.workspace.infrastructure.kubernetes.provision.AsyncStorageProvisioner;
import org.eclipse.che.workspace.infrastructure.kubernetes.provision.CertificateProvisioner;
import org.eclipse.che.workspace.infrastructure.kubernetes.provision.GatewayRouterProvisioner;
import org.eclipse.che.workspace.infrastructure.kubernetes.provision.GitConfigProvisioner;
import org.eclipse.che.workspace.infrastructure.kubernetes.provision.ImagePullSecretProvisioner;
import org.eclipse.che.workspace.infrastructure.kubernetes.provision.KubernetesTrustedCAProvisioner;
import org.eclipse.che.workspace.infrastructure.kubernetes.provision.LogsVolumeMachineProvisioner;
import org.eclipse.che.workspace.infrastructure.kubernetes.provision.NodeSelectorProvisioner;
import org.eclipse.che.workspace.infrastructure.kubernetes.provision.PodTerminationGracePeriodProvisioner;
import org.eclipse.che.workspace.infrastructure.kubernetes.provision.ProxySettingsProvisioner;
@ -68,7 +65,6 @@ public interface KubernetesEnvironmentProvisioner<T extends KubernetesEnvironmen
private final EnvVarsConverter envVarsConverter;
private final RestartPolicyRewriter restartPolicyRewriter;
private final ContainerResourceProvisioner resourceLimitRequestProvisioner;
private final LogsVolumeMachineProvisioner logsVolumeMachineProvisioner;
private final SecurityContextProvisioner securityContextProvisioner;
private final PodTerminationGracePeriodProvisioner podTerminationGracePeriodProvisioner;
private final TlsProvisioner<KubernetesEnvironment> externalServerTlsProvisioner;
@ -76,8 +72,6 @@ public interface KubernetesEnvironmentProvisioner<T extends KubernetesEnvironmen
private final ProxySettingsProvisioner proxySettingsProvisioner;
private final NodeSelectorProvisioner nodeSelectorProvisioner;
private final TolerationsProvisioner tolerationsProvisioner;
private final AsyncStorageProvisioner asyncStorageProvisioner;
private final AsyncStoragePodInterceptor asyncStoragePodInterceptor;
private final ServiceAccountProvisioner serviceAccountProvisioner;
private final CertificateProvisioner certificateProvisioner;
private final SshKeysProvisioner sshKeysProvisioner;
@ -94,7 +88,6 @@ public interface KubernetesEnvironmentProvisioner<T extends KubernetesEnvironmen
EnvVarsConverter envVarsConverter,
RestartPolicyRewriter restartPolicyRewriter,
ContainerResourceProvisioner resourceLimitRequestProvisioner,
LogsVolumeMachineProvisioner logsVolumeMachineProvisioner,
SecurityContextProvisioner securityContextProvisioner,
PodTerminationGracePeriodProvisioner podTerminationGracePeriodProvisioner,
TlsProvisionerProvider<KubernetesEnvironment> externalServerTlsProvisionerProvider,
@ -102,8 +95,6 @@ public interface KubernetesEnvironmentProvisioner<T extends KubernetesEnvironmen
ProxySettingsProvisioner proxySettingsProvisioner,
NodeSelectorProvisioner nodeSelectorProvisioner,
TolerationsProvisioner tolerationsProvisioner,
AsyncStorageProvisioner asyncStorageProvisioner,
AsyncStoragePodInterceptor asyncStoragePodInterceptor,
ServiceAccountProvisioner serviceAccountProvisioner,
CertificateProvisioner certificateProvisioner,
SshKeysProvisioner sshKeysProvisioner,
@ -117,7 +108,6 @@ public interface KubernetesEnvironmentProvisioner<T extends KubernetesEnvironmen
this.envVarsConverter = envVarsConverter;
this.restartPolicyRewriter = restartPolicyRewriter;
this.resourceLimitRequestProvisioner = resourceLimitRequestProvisioner;
this.logsVolumeMachineProvisioner = logsVolumeMachineProvisioner;
this.securityContextProvisioner = securityContextProvisioner;
this.podTerminationGracePeriodProvisioner = podTerminationGracePeriodProvisioner;
this.externalServerTlsProvisioner = externalServerTlsProvisionerProvider.get();
@ -125,8 +115,6 @@ public interface KubernetesEnvironmentProvisioner<T extends KubernetesEnvironmen
this.proxySettingsProvisioner = proxySettingsProvisioner;
this.nodeSelectorProvisioner = nodeSelectorProvisioner;
this.tolerationsProvisioner = tolerationsProvisioner;
this.asyncStorageProvisioner = asyncStorageProvisioner;
this.asyncStoragePodInterceptor = asyncStoragePodInterceptor;
this.serviceAccountProvisioner = serviceAccountProvisioner;
this.certificateProvisioner = certificateProvisioner;
this.sshKeysProvisioner = sshKeysProvisioner;
@ -163,7 +151,6 @@ public interface KubernetesEnvironmentProvisioner<T extends KubernetesEnvironmen
imagePullSecretProvisioner.provision(k8sEnv, identity);
proxySettingsProvisioner.provision(k8sEnv, identity);
serviceAccountProvisioner.provision(k8sEnv, identity);
asyncStorageProvisioner.provision(k8sEnv, identity);
certificateProvisioner.provision(k8sEnv, identity);
sshKeysProvisioner.provision(k8sEnv, identity);
vcsSslCertificateProvisioner.provision(k8sEnv, identity);

3
infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesInfraModule.java
View File

@ -54,7 +54,6 @@ import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.configurato
import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.configurator.UserPreferencesConfigurator;
import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.configurator.UserProfileConfigurator;
import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.configurator.WorkspaceServiceAccountConfigurator;
import org.eclipse.che.workspace.infrastructure.kubernetes.provision.AsyncStorageProvisioner;
import org.eclipse.che.workspace.infrastructure.kubernetes.provision.GatewayTlsProvisioner;
import org.eclipse.che.workspace.infrastructure.kubernetes.provision.IngressTlsProvisioner;
import org.eclipse.che.workspace.infrastructure.kubernetes.provision.KubernetesCheApiExternalEnvVarProvider;
@ -96,7 +95,6 @@ public class KubernetesInfraModule extends AbstractModule {
Multibinder<WorkspaceAttributeValidator> workspaceAttributeValidators =
Multibinder.newSetBinder(binder(), WorkspaceAttributeValidator.class);
workspaceAttributeValidators.addBinding().to(K8sInfraNamespaceWsAttributeValidator.class);
workspaceAttributeValidators.addBinding().to(AsyncStorageModeValidator.class);
// order matters here!
// We first need to grant permissions to user, only then we can run other configurators with
@ -249,6 +247,5 @@ public class KubernetesInfraModule extends AbstractModule {
binder(), KubernetesEnvironment.TYPE);
bind(NonTlsDistributedClusterModeNotifier.class);
bind(AsyncStorageProvisioner.class);
}
}

298
infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/pvc/CommonPVCStrategy.java
View File

@ -1,298 +0,0 @@
/*
* Copyright (c) 2012-2022 Red Hat, Inc.
* This program and the accompanying materials are made
* available under the terms of the Eclipse Public License 2.0
* which is available at https://www.eclipse.org/legal/epl-2.0/
*
* SPDX-License-Identifier: EPL-2.0
*
* Contributors:
* Red Hat, Inc. - initial API and implementation
*/
package org.eclipse.che.workspace.infrastructure.kubernetes.namespace.pvc;
import static com.google.common.base.Strings.isNullOrEmpty;
import static java.lang.String.format;
import static java.util.stream.Collectors.joining;
import static java.util.stream.Collectors.toSet;
import static org.eclipse.che.api.user.server.UserManager.PERSONAL_ACCOUNT;
import static org.eclipse.che.workspace.infrastructure.kubernetes.namespace.KubernetesObjectUtil.newPVC;
import com.google.inject.Inject;
import io.fabric8.kubernetes.api.model.PersistentVolumeClaim;
import io.fabric8.kubernetes.api.model.VolumeMount;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import org.eclipse.che.account.spi.AccountImpl;
import org.eclipse.che.api.core.Page;
import org.eclipse.che.api.core.ServerException;
import org.eclipse.che.api.core.model.workspace.Workspace;
import org.eclipse.che.api.core.model.workspace.runtime.RuntimeIdentity;
import org.eclipse.che.api.workspace.server.WorkspaceManager;
import org.eclipse.che.api.workspace.server.model.impl.WorkspaceImpl;
import org.eclipse.che.api.workspace.server.spi.InfrastructureException;
import org.eclipse.che.commons.annotation.Traced;
import org.eclipse.che.commons.tracing.TracingTags;
import org.eclipse.che.workspace.infrastructure.kubernetes.environment.KubernetesEnvironment;
import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.KubernetesNamespace;
import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.KubernetesNamespaceFactory;
import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.KubernetesPersistentVolumeClaims;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Provides common PVC for all workspaces in one Kubernetes namespace.
*
* <p>This strategy indirectly affects the workspace limits.<br>
* The number of workspaces that can simultaneously store backups in one PV is limited only by the
* storage capacity. The number of workspaces that can be running simultaneously depends on access
* mode configuration and Che configuration limits.
*
* <p><b>Used subpaths:</b>
*
* <p>This strategy uses subpaths for resolving backed up data paths collisions.<br>
* Subpaths have the following format: '{workspaceId}/{volumeName}'.<br>
* Note that logs volume has the special format: '{workspaceId}/{volumeName}/{machineName}'. It is
* done in this way to avoid conflicts e.g. two identical agents inside different machines produce
* the same log file.
*
* <p><b>How user-defined PVCs are processed:</b>
*
* <p>How user-defined PVCs are processed: User-defined PVCs are removed from environment. Pods
* volumes that reference PVCs are replaced with volume that references common PVC. The
* corresponding containers volume mounts are relinked to common volume and subpaths are prefixed
* with `'{workspaceId}/{originalPVCName}'`.
*
* <p>User-defined PVC name is used as Che Volume name. It means that if Machine is configured to
* use Che Volume with the same name as user-defined PVC has then they will use the same shared
* folder in common PVC.
*
* <p>Note that quantity and access mode of user-defined PVCs are ignored since common PVC is used
* and it has preconfigured configuration.
*
* @author Anton Korneta
* @author Alexander Garagatyi
*/
public class CommonPVCStrategy implements WorkspaceVolumesStrategy {
// use non-static variable to reuse child class logger
private final Logger log = LoggerFactory.getLogger(getClass());
public static final String COMMON_STRATEGY = "common";
/**
* The additional property name with the wildcard reserved for workspace id. Formatted property
* with the real workspace id is used to get workspace subpaths directories. The value of this
* property represents the String array of subpaths that are used to create folders in PV with
* user rights. Note that the value would not be stored and it is removed before PVC creation.
*/
static final String SUBPATHS_PROPERTY_FMT = "che.workspace.%s.subpaths";
private final boolean preCreateDirs;
private final String pvcQuantity;
private final String configuredPVCName;
private final String pvcAccessMode;
private final String pvcStorageClassName;
private final PVCSubPathHelper pvcSubPathHelper;
private final KubernetesNamespaceFactory factory;
private final EphemeralWorkspaceAdapter ephemeralWorkspaceAdapter;
private final PVCProvisioner pvcProvisioner;
private final PodsVolumes podsVolumes;
private final SubPathPrefixes subpathPrefixes;
private final boolean waitBound;
private final WorkspaceManager workspaceManager;
@Inject
public CommonPVCStrategy(
PVCSubPathHelper pvcSubPathHelper,
KubernetesNamespaceFactory factory,
EphemeralWorkspaceAdapter ephemeralWorkspaceAdapter,
PVCProvisioner pvcProvisioner,
PodsVolumes podsVolumes,
SubPathPrefixes subpathPrefixes,
WorkspaceManager workspaceManager) {
this.configuredPVCName = "test";
this.pvcQuantity = "test";
this.pvcAccessMode = "TEST";
this.preCreateDirs = true;
this.pvcStorageClassName = "TEST";
this.waitBound = true;
this.pvcSubPathHelper = pvcSubPathHelper;
this.factory = factory;
this.ephemeralWorkspaceAdapter = ephemeralWorkspaceAdapter;
this.pvcProvisioner = pvcProvisioner;
this.podsVolumes = podsVolumes;
this.subpathPrefixes = subpathPrefixes;
this.workspaceManager = workspaceManager;
}
/**
* Creates new instance of PVC object that should be used for the specified workspace.
*
* <p>May be overridden by child class for changing common scope. Like common per user or common
* per workspace.
*
* @param workspaceId workspace that needs PVC
* @return pvc that should be used for the specified runtime identity
*/
protected PersistentVolumeClaim createCommonPVC(String workspaceId) {
return newPVC(configuredPVCName, pvcAccessMode, pvcQuantity, pvcStorageClassName);
}
@Override
public void provision(KubernetesEnvironment k8sEnv, RuntimeIdentity identity)
throws InfrastructureException {
final String workspaceId = identity.getWorkspaceId();
if (EphemeralWorkspaceUtility.isEphemeral(k8sEnv.getAttributes())) {
ephemeralWorkspaceAdapter.provision(k8sEnv, identity);
return;
}
log.debug("Provisioning PVC strategy for workspace '{}'", workspaceId);
pvcProvisioner.convertCheVolumes(k8sEnv, workspaceId);
// Note that PVC name is used during prefixing
// It MUST be done before changing all PVCs references to common PVC
subpathPrefixes.prefixVolumeMountsSubpaths(k8sEnv, identity.getWorkspaceId());
PersistentVolumeClaim commonPVC = replacePVCsWithCommon(k8sEnv, identity);
podsVolumes.replacePVCVolumesWithCommon(
k8sEnv.getPodsData(), commonPVC.getMetadata().getName());
if (preCreateDirs) {
Set<String> subPaths = combineVolumeMountsSubpaths(k8sEnv);
if (!subPaths.isEmpty()) {
commonPVC.setAdditionalProperty(
format(SUBPATHS_PROPERTY_FMT, workspaceId),
subPaths.toArray(new String[subPaths.size()]));
}
}
log.debug("PVC strategy provisioning done for workspace '{}'", workspaceId);
}
@Override
@Traced
public void prepare(
KubernetesEnvironment k8sEnv,
RuntimeIdentity identity,
long timeoutMillis,
Map<String, String> startOptions)
throws InfrastructureException {
String workspaceId = identity.getWorkspaceId();
TracingTags.WORKSPACE_ID.set(workspaceId);
if (EphemeralWorkspaceUtility.isEphemeral(k8sEnv.getAttributes())) {
return;
}
log.debug("Preparing PVC started for workspace '{}'", workspaceId);
Map<String, PersistentVolumeClaim> claims = k8sEnv.getPersistentVolumeClaims();
if (claims.isEmpty()) {
return;
}
if (claims.size() > 1) {
throw new InfrastructureException(
format(
"The only one PVC MUST be present in common strategy while it contains: %s.",
claims.keySet().stream().collect(joining(", "))));
}
PersistentVolumeClaim commonPVC = claims.values().iterator().next();
final KubernetesNamespace namespace = factory.getOrCreate(identity);
final KubernetesPersistentVolumeClaims pvcs = namespace.persistentVolumeClaims();
final Set<String> existing =
pvcs.get().stream().map(p -> p.getMetadata().getName()).collect(toSet());
if (!existing.contains(commonPVC.getMetadata().getName())) {
log.debug("Creating PVC for workspace '{}'", workspaceId);
pvcs.create(commonPVC);
if (waitBound) {
log.debug("Waiting for PVC for workspace '{}' to be bound", workspaceId);
pvcs.waitBound(commonPVC.getMetadata().getName(), timeoutMillis);
}
}
final String[] subpaths =
(String[])
commonPVC.getAdditionalProperties().remove(format(SUBPATHS_PROPERTY_FMT, workspaceId));
if (preCreateDirs && subpaths != null) {
pvcSubPathHelper.createDirs(
identity, workspaceId, commonPVC.getMetadata().getName(), startOptions, subpaths);
}
log.debug("Preparing PVC done for workspace '{}'", workspaceId);
}
@Override
public void cleanup(Workspace workspace) throws InfrastructureException {
AccountImpl account = ((WorkspaceImpl) workspace).getAccount();
if (isPersonalAccount(account) && accountHasNoWorkspaces(account)) {
log.debug("Deleting the common PVC: '{}',", configuredPVCName);
deleteCommonPVC(workspace);
return;
}
if (EphemeralWorkspaceUtility.isEphemeral(workspace)) {
return;
}
String workspaceId = workspace.getId();
PersistentVolumeClaim pvc = createCommonPVC(workspaceId);
pvcSubPathHelper.removeDirsAsync(
workspaceId,
factory.get(workspace).getName(),
pvc.getMetadata().getName(),
subpathPrefixes.getWorkspaceSubPath(workspaceId));
}
private PersistentVolumeClaim replacePVCsWithCommon(
KubernetesEnvironment k8sEnv, RuntimeIdentity identity) {
final PersistentVolumeClaim commonPVC = createCommonPVC(identity.getWorkspaceId());
k8sEnv.getPersistentVolumeClaims().clear();
k8sEnv.getPersistentVolumeClaims().put(commonPVC.getMetadata().getName(), commonPVC);
return commonPVC;
}
private Set<String> combineVolumeMountsSubpaths(KubernetesEnvironment k8sEnv) {
return k8sEnv.getPodsData().values().stream()
.flatMap(p -> p.getSpec().getContainers().stream())
.flatMap(c -> c.getVolumeMounts().stream())
.map(VolumeMount::getSubPath)
.filter(subpath -> !isNullOrEmpty(subpath))
.collect(Collectors.toSet());
}
private void deleteCommonPVC(Workspace workspace) throws InfrastructureException {
factory.get(workspace).persistentVolumeClaims().delete(configuredPVCName);
}
/**
* @param account the account of interest
* @return true, if the given account is a personal account, false otherwise
*/
private boolean isPersonalAccount(AccountImpl account) {
return PERSONAL_ACCOUNT.equals(account.getType());
}
/**
* @param account the account of interest
* @return true, if the given account has no workspaces, false otherwise
* @throws InfrastructureException
*/
private boolean accountHasNoWorkspaces(AccountImpl account) throws InfrastructureException {
try {
Page<WorkspaceImpl> workspaces = workspaceManager.getWorkspaces(account.getId(), false, 1, 0);
if (workspaces.isEmpty()) {
log.debug("User '{}' has no more workspaces left", account.getId());
return true;
}
} catch (ServerException e) {
// should never happen
throw new InfrastructureException(e.getLocalizedMessage(), e);
}
return false;
}
}

82
infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/pvc/EphemeralWorkspaceAdapter.java
View File

@ -1,82 +0,0 @@
/*
* Copyright (c) 2012-2021 Red Hat, Inc.
* This program and the accompanying materials are made
* available under the terms of the Eclipse Public License 2.0
* which is available at https://www.eclipse.org/legal/epl-2.0/
*
* SPDX-License-Identifier: EPL-2.0
*
* Contributors:
* Red Hat, Inc. - initial API and implementation
*/
package org.eclipse.che.workspace.infrastructure.kubernetes.namespace.pvc;
import io.fabric8.kubernetes.api.model.EmptyDirVolumeSource;
import io.fabric8.kubernetes.api.model.PersistentVolumeClaim;
import io.fabric8.kubernetes.api.model.PodSpec;
import java.util.HashMap;
import java.util.Map;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.eclipse.che.api.core.model.workspace.runtime.RuntimeIdentity;
import org.eclipse.che.api.workspace.server.spi.InfrastructureException;
import org.eclipse.che.workspace.infrastructure.kubernetes.environment.KubernetesEnvironment;
import org.eclipse.che.workspace.infrastructure.kubernetes.environment.KubernetesEnvironment.PodData;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Allows to create ephemeral workspaces (with no PVC attached) based on workspace config
* `persistVolumes` attribute. If `persistVolumes` attribute is set to false, workspace volumes
* would be created as `emptyDir` regardless of the PVC strategy. User-defined PVCs will be removed
* from environment and the corresponding PVC volumes in Pods will be replaced with `emptyDir`
* volumes. When a workspace Pod is removed for any reason, the data in the `emptyDir` volume is
* deleted forever.
*
* @see <a href="https://kubernetes.io/docs/concepts/storage/volumes/#emptydir">emptyDir</a>
* @author Ilya Buziuk
* @author Angel Misevski
*/
@Singleton
public class EphemeralWorkspaceAdapter {
private static final Logger LOG = LoggerFactory.getLogger(CommonPVCStrategy.class);
private final PVCProvisioner pvcProvisioner;
private final SubPathPrefixes subPathPrefixes;
@Inject
public EphemeralWorkspaceAdapter(PVCProvisioner pvcProvisioner, SubPathPrefixes subPathPrefixes) {
this.pvcProvisioner = pvcProvisioner;
this.subPathPrefixes = subPathPrefixes;
}
public void provision(KubernetesEnvironment k8sEnv, RuntimeIdentity identity)
throws InfrastructureException {
LOG.debug("Provisioning PVC strategy for workspace '{}'", identity.getWorkspaceId());
Map<String, PersistentVolumeClaim> userDefinedPVCs =
new HashMap<>(k8sEnv.getPersistentVolumeClaims());
k8sEnv.getPersistentVolumeClaims().clear();
pvcProvisioner.provision(k8sEnv, userDefinedPVCs);
pvcProvisioner.convertCheVolumes(k8sEnv, identity.getWorkspaceId());
subPathPrefixes.prefixVolumeMountsSubpaths(k8sEnv, identity.getWorkspaceId());
replacePVCsWithEmptyDir(k8sEnv);
k8sEnv.getPersistentVolumeClaims().clear();
}
private void replacePVCsWithEmptyDir(KubernetesEnvironment k8sEnv) {
for (PodData pod : k8sEnv.getPodsData().values()) {
PodSpec podSpec = pod.getSpec();
podSpec.getVolumes().stream()
.filter(v -> v.getPersistentVolumeClaim() != null)
.forEach(
v -> {
v.setPersistentVolumeClaim(null);
v.setEmptyDir(new EmptyDirVolumeSource());
});
}
}
}

97
infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/pvc/PerWorkspacePVCStrategy.java
View File

@ -1,97 +0,0 @@
/*
* Copyright (c) 2012-2022 Red Hat, Inc.
* This program and the accompanying materials are made
* available under the terms of the Eclipse Public License 2.0
* which is available at https://www.eclipse.org/legal/epl-2.0/
*
* SPDX-License-Identifier: EPL-2.0
*
* Contributors:
* Red Hat, Inc. - initial API and implementation
*/
package org.eclipse.che.workspace.infrastructure.kubernetes.namespace.pvc;
import static org.eclipse.che.workspace.infrastructure.kubernetes.Constants.CHE_WORKSPACE_ID_LABEL;
import static org.eclipse.che.workspace.infrastructure.kubernetes.namespace.KubernetesObjectUtil.newPVC;
import static org.eclipse.che.workspace.infrastructure.kubernetes.namespace.KubernetesObjectUtil.putLabel;
import com.google.common.collect.ImmutableMap;
import io.fabric8.kubernetes.api.model.PersistentVolumeClaim;
import javax.inject.Inject;
import org.eclipse.che.api.core.model.workspace.Workspace;
import org.eclipse.che.api.workspace.server.WorkspaceManager;
import org.eclipse.che.api.workspace.server.spi.InfrastructureException;
import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.KubernetesNamespaceFactory;
/**
* Provides common PVC per each workspace.
*
* <p>Names for PVCs are evaluated as: '{configured_prefix}' + '-' +'{workspaceId}' to avoid naming
* collisions inside of one Kubernetes namespace.
*
* <p>This strategy uses subpaths to do the same as {@link CommonPVCStrategy} does and make easier
* data migration if it will be needed.<br>
* Subpaths have the following format: '{workspaceId}/{volumeName}'.<br>
* Note that logs volume has the special format: '{workspaceId}/{volumeName}/{machineName}'. It is
* done in this way to avoid conflicts e.g. two identical agents inside different machines produce
* the same log file.
*
* @author Sergii Leshchenko
* @author Masaki Muranaka
*/
public class PerWorkspacePVCStrategy extends CommonPVCStrategy {
public static final String PER_WORKSPACE_STRATEGY = "per-workspace";
private final KubernetesNamespaceFactory factory;
private final String pvcNamePrefix;
private final String pvcAccessMode;
private final String pvcQuantity;
private final String pvcStorageClassName;
@Inject
public PerWorkspacePVCStrategy(
PVCSubPathHelper pvcSubPathHelper,
KubernetesNamespaceFactory factory,
EphemeralWorkspaceAdapter ephemeralWorkspaceAdapter,
PVCProvisioner pvcProvisioner,
PodsVolumes podsVolumes,
SubPathPrefixes subpathPrefixes,
WorkspaceManager workspaceManager) {
super(
pvcSubPathHelper,
factory,
ephemeralWorkspaceAdapter,
pvcProvisioner,
podsVolumes,
subpathPrefixes,
workspaceManager);
this.pvcNamePrefix = "TEST";
this.factory = factory;
this.pvcAccessMode = "TEST";
this.pvcQuantity = "test";
this.pvcStorageClassName = "TEST";
}
@Override
protected PersistentVolumeClaim createCommonPVC(String workspaceId) {
String pvcName = pvcNamePrefix + '-' + workspaceId;
PersistentVolumeClaim perWorkspacePVC =
newPVC(pvcName, pvcAccessMode, pvcQuantity, pvcStorageClassName);
putLabel(perWorkspacePVC.getMetadata(), CHE_WORKSPACE_ID_LABEL, workspaceId);
return perWorkspacePVC;
}
@Override
public void cleanup(Workspace workspace) throws InfrastructureException {
if (EphemeralWorkspaceUtility.isEphemeral(workspace)) {
return;
}
final String workspaceId = workspace.getId();
factory
.get(workspace)
.persistentVolumeClaims()
.delete(ImmutableMap.of(CHE_WORKSPACE_ID_LABEL, workspaceId));
}
}

190
infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/pvc/UniqueWorkspacePVCStrategy.java
View File

@ -1,190 +0,0 @@
/*
* Copyright (c) 2012-2022 Red Hat, Inc.
* This program and the accompanying materials are made
* available under the terms of the Eclipse Public License 2.0
* which is available at https://www.eclipse.org/legal/epl-2.0/
*
* SPDX-License-Identifier: EPL-2.0
*
* Contributors:
* Red Hat, Inc. - initial API and implementation
*/
package org.eclipse.che.workspace.infrastructure.kubernetes.namespace.pvc;
import static java.util.stream.Collectors.toMap;
import static org.eclipse.che.workspace.infrastructure.kubernetes.Constants.CHE_WORKSPACE_ID_LABEL;
import com.google.common.collect.ImmutableMap;
import io.fabric8.kubernetes.api.model.PersistentVolumeClaim;
import java.util.HashMap;
import java.util.Map;
import java.util.function.Function;
import javax.inject.Inject;
import org.eclipse.che.api.core.model.workspace.Workspace;
import org.eclipse.che.api.core.model.workspace.runtime.RuntimeIdentity;
import org.eclipse.che.api.workspace.server.spi.InfrastructureException;
import org.eclipse.che.commons.annotation.Traced;
import org.eclipse.che.commons.tracing.TracingTags;
import org.eclipse.che.workspace.infrastructure.kubernetes.environment.KubernetesEnvironment;
import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.KubernetesNamespaceFactory;
import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.KubernetesPersistentVolumeClaims;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Provides a unique PVC for each volume of a workspace.
*
* <p>Names for PVCs are evaluated as: '{configured_prefix}' + '-' +'{generated_8_chars}' to avoid
* naming collisions inside of one Kubernetes namespace.
*
* <p>Note that for in this strategy number of simultaneously used volumes by workspaces can not be
* greater than the number of available PVCs in Kubernetes namespace.
*
* <p><b>Used subpaths:</b>
*
* <p>This strategy uses subpaths to do the same as {@link CommonPVCStrategy} does and make easier
* data migration if it will be needed.<br>
* Subpaths have the following format: '{workspaceId}/{volume/PVC name}'.<br>
* Note that logs volume has the special format: '{workspaceId}/{volumeName}/{machineName}'. It is
* done in this way to avoid conflicts e.g. two identical agents inside different machines produce
* the same log file.
*
* <p><b>How user-defined PVCs are processed:</b>
*
* <p>User-defined PVCs are provisioned with generated unique names. Pods volumes that reference
* PVCs are updated accordingly. Subpaths of the corresponding containers volume mounts are prefixed
* with `'{workspaceId}/{originalPVCName}'`.
*
* <p>User-defined PVC name is used as Che Volume name. It means that if Machine is configured to
* use Che Volume with the same name as user-defined PVC has then Che Volume will reuse user-defined
* PVC.
*
* <p>Note that quantity and access mode of user-defined PVCs are not overridden with Che Server
* configured.
*
* <p><b>Clean up:</b>
*
* <p>Cleanup of backed up data is performed by removing of PVCs related to the workspace but when
* the volume or machine name is changed then related PVC would not be removed.
*
* @author Anton Korneta
* @author Alexander Garagatyi
*/
public class UniqueWorkspacePVCStrategy implements WorkspaceVolumesStrategy {
private static final Logger LOG = LoggerFactory.getLogger(UniqueWorkspacePVCStrategy.class);
public static final String UNIQUE_STRATEGY = "unique";
private final KubernetesNamespaceFactory factory;
private final EphemeralWorkspaceAdapter ephemeralWorkspaceAdapter;
private final PVCProvisioner pvcProvisioner;
private final SubPathPrefixes subpathPrefixes;
private final boolean waitBound;
@Inject
public UniqueWorkspacePVCStrategy(
KubernetesNamespaceFactory factory,
EphemeralWorkspaceAdapter ephemeralWorkspaceAdapter,
PVCProvisioner pvcProvisioner,
SubPathPrefixes subpathPrefixes) {
this.waitBound = true;
this.factory = factory;
this.ephemeralWorkspaceAdapter = ephemeralWorkspaceAdapter;
this.pvcProvisioner = pvcProvisioner;
this.subpathPrefixes = subpathPrefixes;
}
@Override
public void provision(KubernetesEnvironment k8sEnv, RuntimeIdentity identity)
throws InfrastructureException {
final String workspaceId = identity.getWorkspaceId();
if (EphemeralWorkspaceUtility.isEphemeral(k8sEnv.getAttributes())) {
ephemeralWorkspaceAdapter.provision(k8sEnv, identity);
return;
}
LOG.debug("Provisioning PVC strategy for workspace '{}'", workspaceId);
Map<String, PersistentVolumeClaim> userDefinedPVCs =
new HashMap<>(k8sEnv.getPersistentVolumeClaims());
k8sEnv.getPersistentVolumeClaims().clear();
fillInExistingPVCs(k8sEnv, identity);
pvcProvisioner.provision(k8sEnv, userDefinedPVCs);
pvcProvisioner.convertCheVolumes(k8sEnv, workspaceId);
subpathPrefixes.prefixVolumeMountsSubpaths(k8sEnv, identity.getWorkspaceId());
provisionWorkspaceIdLabel(k8sEnv.getPersistentVolumeClaims(), identity.getWorkspaceId());
LOG.debug("PVC strategy provisioning done for workspace '{}'", workspaceId);
}
@Traced
@Override
public void prepare(
KubernetesEnvironment k8sEnv,
RuntimeIdentity identity,
long timeoutMillis,
Map<String, String> startOptions)
throws InfrastructureException {
String workspaceId = identity.getWorkspaceId();
TracingTags.WORKSPACE_ID.set(workspaceId);
if (EphemeralWorkspaceUtility.isEphemeral(k8sEnv.getAttributes())) {
return;
}
if (k8sEnv.getPersistentVolumeClaims().isEmpty()) {
// no PVCs to prepare
return;
}
final KubernetesPersistentVolumeClaims k8sClaims =
factory.getOrCreate(identity).persistentVolumeClaims();
LOG.debug("Creating PVCs for workspace '{}'", workspaceId);
k8sClaims.createIfNotExist(k8sEnv.getPersistentVolumeClaims().values());
if (waitBound) {
LOG.debug("Waiting for PVC(s) of workspace '{}' to be bound", workspaceId);
for (PersistentVolumeClaim pvc : k8sEnv.getPersistentVolumeClaims().values()) {
k8sClaims.waitBound(pvc.getMetadata().getName(), timeoutMillis);
}
}
LOG.debug("Preparing PVCs done for workspace '{}'", workspaceId);
}
@Override
public void cleanup(Workspace workspace) throws InfrastructureException {
if (EphemeralWorkspaceUtility.isEphemeral(workspace)) {
return;
}
factory
.get(workspace)
.persistentVolumeClaims()
.delete(ImmutableMap.of(CHE_WORKSPACE_ID_LABEL, workspace.getId()));
}
private void fillInExistingPVCs(KubernetesEnvironment k8sEnv, RuntimeIdentity identity)
throws InfrastructureException {
Map<String, PersistentVolumeClaim> existingPVCs =
factory
.getOrCreate(identity)
.persistentVolumeClaims()
.getByLabel(CHE_WORKSPACE_ID_LABEL, identity.getWorkspaceId())
.stream()
.collect(toMap(pvc -> pvc.getMetadata().getName(), Function.identity()));
k8sEnv.getPersistentVolumeClaims().putAll(existingPVCs);
}
private void provisionWorkspaceIdLabel(
Map<String, PersistentVolumeClaim> pvcs, String workspaceId) {
pvcs.values()
.forEach(pvc -> pvc.getMetadata().getLabels().put(CHE_WORKSPACE_ID_LABEL, workspaceId));
}
}

222
infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/AsyncStoragePodInterceptor.java
View File

@ -1,222 +0,0 @@
/*
* Copyright (c) 2012-2022 Red Hat, Inc.
* This program and the accompanying materials are made
* available under the terms of the Eclipse Public License 2.0
* which is available at https://www.eclipse.org/legal/epl-2.0/
*
* SPDX-License-Identifier: EPL-2.0
*
* Contributors:
* Red Hat, Inc. - initial API and implementation
*/
package org.eclipse.che.workspace.infrastructure.kubernetes.provision;
import static io.fabric8.kubernetes.api.model.DeletionPropagation.BACKGROUND;
import static java.lang.String.format;
import static org.eclipse.che.workspace.infrastructure.kubernetes.namespace.pvc.CommonPVCStrategy.COMMON_STRATEGY;
import static org.eclipse.che.workspace.infrastructure.kubernetes.namespace.pvc.EphemeralWorkspaceUtility.isEphemeral;
import static org.eclipse.che.workspace.infrastructure.kubernetes.provision.AsyncStorageProvisioner.ASYNC_STORAGE;
import io.fabric8.kubernetes.api.model.Pod;
import io.fabric8.kubernetes.api.model.apps.Deployment;
import io.fabric8.kubernetes.client.KubernetesClientException;
import io.fabric8.kubernetes.client.Watch;
import io.fabric8.kubernetes.client.Watcher;
import io.fabric8.kubernetes.client.WatcherException;
import io.fabric8.kubernetes.client.dsl.PodResource;
import io.fabric8.kubernetes.client.dsl.RollableScalableResource;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.eclipse.che.api.core.model.workspace.runtime.RuntimeIdentity;
import org.eclipse.che.api.workspace.server.spi.InfrastructureException;
import org.eclipse.che.workspace.infrastructure.kubernetes.KubernetesClientFactory;
import org.eclipse.che.workspace.infrastructure.kubernetes.KubernetesInfrastructureException;
import org.eclipse.che.workspace.infrastructure.kubernetes.environment.KubernetesEnvironment;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* This interceptor checks whether the starting workspace is configured with persistent storage and
* makes sure to stop the async storage deployment (if any is running) to prevent "Multi-Attach
* error for volume". After the async storage deployment is stopped and deleted, the workspace start
* is resumed.
*/
@Singleton
public class AsyncStoragePodInterceptor {
private static final Logger LOG = LoggerFactory.getLogger(AsyncStoragePodInterceptor.class);
private static final int DELETE_DEPLOYMENT_TIMEOUT_IN_MIN = 5;
private final KubernetesClientFactory kubernetesClientFactory;
private final String pvcStrategy;
@Inject
public AsyncStoragePodInterceptor(KubernetesClientFactory kubernetesClientFactory) {
this.pvcStrategy = "TEST";
this.kubernetesClientFactory = kubernetesClientFactory;
}
public void intercept(KubernetesEnvironment k8sEnv, RuntimeIdentity identity)
throws InfrastructureException {
if (!COMMON_STRATEGY.equals(pvcStrategy) || isEphemeral(k8sEnv.getAttributes())) {
return;
}
removeAsyncStoragePodWithoutDeployment(identity);
String namespace = identity.getInfrastructureNamespace();
String workspaceId = identity.getWorkspaceId();
RollableScalableResource<Deployment> asyncStorageDeploymentResource =
getAsyncStorageDeploymentResource(namespace, workspaceId);
if (asyncStorageDeploymentResource.get() == null) { // deployment doesn't exist
return;
}
try {
deleteAsyncStorageDeployment(asyncStorageDeploymentResource)
.get(DELETE_DEPLOYMENT_TIMEOUT_IN_MIN, TimeUnit.MINUTES);
} catch (InterruptedException ex) {
Thread.currentThread().interrupt();
throw new InfrastructureException(
format(
"Interrupted while waiting for deployment '%s' removal. " + ex.getMessage(),
ASYNC_STORAGE),
ex);
} catch (ExecutionException ex) {
throw new InfrastructureException(
format(
"Error occurred while waiting for deployment '%s' removal. " + ex.getMessage(),
ASYNC_STORAGE),
ex);
} catch (TimeoutException ex) {
throw new InfrastructureException(
format("Pod '%s' removal timeout reached " + ex.getMessage(), ASYNC_STORAGE), ex);
}
}
private RollableScalableResource<Deployment> getAsyncStorageDeploymentResource(
String namespace, String workspaceId) throws InfrastructureException {
return kubernetesClientFactory
.create(workspaceId)
.apps()
.deployments()
.inNamespace(namespace)
.withName(ASYNC_STORAGE);
}
private CompletableFuture<Void> deleteAsyncStorageDeployment(
RollableScalableResource<Deployment> resource) throws InfrastructureException {
Watch toCloseOnException = null;
try {
final CompletableFuture<Void> deleteFuture = new CompletableFuture<>();
final Watch watch = resource.watch(new DeleteWatcher<>(deleteFuture));
toCloseOnException = watch;
Boolean deleteSucceeded = resource.withPropagationPolicy(BACKGROUND).delete();
if (deleteSucceeded == null || !deleteSucceeded) {
deleteFuture.complete(null);
}
return deleteFuture.whenComplete(
(v, e) -> {
if (e != null) {
LOG.warn("Failed to remove deployment {} cause {}", ASYNC_STORAGE, e.getMessage());
}
watch.close();
});
} catch (KubernetesClientException e) {
if (toCloseOnException != null) {
toCloseOnException.close();
}
throw new KubernetesInfrastructureException(e);
} catch (Exception e) {
if (toCloseOnException != null) {
toCloseOnException.close();
}
throw e;
}
}
/**
* Cleanup existed Async Storage pods running without Deployment see
* https://github.com/eclipse/che/issues/17616. Method can be removed in 7.20.x
*/
private void removeAsyncStoragePodWithoutDeployment(RuntimeIdentity identity)
throws InfrastructureException {
String namespace = identity.getInfrastructureNamespace();
String workspaceId = identity.getWorkspaceId();
PodResource<Pod> asyncStoragePodResource =
kubernetesClientFactory
.create(workspaceId)
.pods()
.inNamespace(namespace)
.withName(ASYNC_STORAGE);
if (asyncStoragePodResource.get()
!= null) { // remove existed pod to replace it with deployment on provision step
deleteAsyncStoragePod(asyncStoragePodResource);
}
}
private CompletableFuture<Void> deleteAsyncStoragePod(PodResource<Pod> podResource)
throws InfrastructureException {
Watch toCloseOnException = null;
try {
final CompletableFuture<Void> deleteFuture = new CompletableFuture<>();
final Watch watch = podResource.watch(new DeleteWatcher<>(deleteFuture));
toCloseOnException = watch;
Boolean deleteSucceeded = podResource.withPropagationPolicy(BACKGROUND).delete();
if (deleteSucceeded == null || !deleteSucceeded) {
deleteFuture.complete(null);
}
return deleteFuture.whenComplete(
(v, e) -> {
if (e != null) {
LOG.warn("Failed to remove pod {} cause {}", ASYNC_STORAGE, e.getMessage());
}
watch.close();
});
} catch (KubernetesClientException e) {
if (toCloseOnException != null) {
toCloseOnException.close();
}
throw new KubernetesInfrastructureException(e);
} catch (Exception e) {
if (toCloseOnException != null) {
toCloseOnException.close();
}
throw e;
}
}
private static class DeleteWatcher<T> implements Watcher<T> {
private final CompletableFuture<Void> future;
private DeleteWatcher(CompletableFuture<Void> future) {
this.future = future;
}
@Override
public void eventReceived(Action action, T hasMetadata) {
if (action == Action.DELETED) {
future.complete(null);
}
}
@Override
public void onClose(WatcherException e) {
// if event about removing is received then this completion has no effect
future.completeExceptionally(
new RuntimeException(
"WebSocket connection is closed. But event about removing is not received.", e));
}
}
}

165
infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/AsyncStoragePodWatcher.java
View File

@ -1,165 +0,0 @@
/*
* Copyright (c) 2012-2022 Red Hat, Inc.
* This program and the accompanying materials are made
* available under the terms of the Eclipse Public License 2.0
* which is available at https://www.eclipse.org/legal/epl-2.0/
*
* SPDX-License-Identifier: EPL-2.0
*
* Contributors:
* Red Hat, Inc. - initial API and implementation
*/
package org.eclipse.che.workspace.infrastructure.kubernetes.provision;
import static com.google.common.base.Strings.isNullOrEmpty;
import static java.lang.Long.parseLong;
import static java.time.Instant.now;
import static java.time.Instant.ofEpochSecond;
import static java.util.concurrent.TimeUnit.MINUTES;
import static org.eclipse.che.api.core.Pages.iterateLazily;
import static org.eclipse.che.api.workspace.shared.Constants.LAST_ACTIVE_INFRASTRUCTURE_NAMESPACE;
import static org.eclipse.che.api.workspace.shared.Constants.LAST_ACTIVITY_TIME;
import static org.eclipse.che.workspace.infrastructure.kubernetes.namespace.pvc.CommonPVCStrategy.COMMON_STRATEGY;
import static org.eclipse.che.workspace.infrastructure.kubernetes.provision.AsyncStorageProvisioner.ASYNC_STORAGE;
import io.fabric8.kubernetes.api.model.Pod;
import io.fabric8.kubernetes.api.model.apps.Deployment;
import io.fabric8.kubernetes.client.dsl.PodResource;
import io.fabric8.kubernetes.client.dsl.RollableScalableResource;
import java.time.Instant;
import java.util.Map;
import javax.inject.Inject;
import javax.inject.Named;
import javax.inject.Singleton;
import org.eclipse.che.api.core.ServerException;
import org.eclipse.che.api.core.model.user.User;
import org.eclipse.che.api.user.server.PreferenceManager;
import org.eclipse.che.api.user.server.UserManager;
import org.eclipse.che.api.workspace.server.WorkspaceRuntimes;
import org.eclipse.che.api.workspace.server.spi.InfrastructureException;
import org.eclipse.che.commons.annotation.Nullable;
import org.eclipse.che.commons.schedule.ScheduleDelay;
import org.eclipse.che.workspace.infrastructure.kubernetes.KubernetesClientFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Periodically checks ability to stop Asynchronous Storage Pod. It will periodically revise
* UserPreferences of all registered user and check specialized preferences. Preferences should be
* recorded if last workspace stopped and cleanup on start any workspace. Required preferences to
* initiate stop procedure for Asynchronous Storage Pod : {@link
* org.eclipse.che.api.workspace.shared.Constants#LAST_ACTIVE_INFRASTRUCTURE_NAMESPACE} : should
* contain last used infrastructure namespace {@link
* org.eclipse.che.api.workspace.shared.Constants#LAST_ACTIVITY_TIME} : seconds then workspace
* stopped in the Java epoch time format (aka Unix time)
*/
@Singleton
public class AsyncStoragePodWatcher {
private static final Logger LOG = LoggerFactory.getLogger(AsyncStoragePodWatcher.class);
private final KubernetesClientFactory kubernetesClientFactory;
private final UserManager userManager;
private final PreferenceManager preferenceManager;
private final WorkspaceRuntimes runtimes;
private final long shutdownTimeoutSec;
private final boolean isAsyncStoragePodCanBeRun;
@Inject
public AsyncStoragePodWatcher(
KubernetesClientFactory kubernetesClientFactory,
UserManager userManager,
PreferenceManager preferenceManager,
WorkspaceRuntimes runtimes,
@Named("che.infra.kubernetes.async.storage.shutdown_timeout_min") long shutdownTimeoutMin,
@Nullable @Named("che.infra.kubernetes.namespace.default") String defaultNamespaceName,
@Named("che.limits.user.workspaces.run.count") int runtimesPerUser) {
this.kubernetesClientFactory = kubernetesClientFactory;
this.userManager = userManager;
this.preferenceManager = preferenceManager;
this.runtimes = runtimes;
this.shutdownTimeoutSec = MINUTES.toSeconds(shutdownTimeoutMin);
isAsyncStoragePodCanBeRun =
isAsyncStoragePodCanBeRun("TEST", defaultNamespaceName, runtimesPerUser);
}
/**
* Checking current system configuration on ability to run Async Storage Pod. Will be checked next
* value of properties:
*
* <ul>
* <li>che.infra.kubernetes.namespace.default=<username>-che
* <li>che.infra.kubernetes.pvc.strategy=common
* <li>che.limits.user.workspaces.run.count=1
* </ul>
*/
private boolean isAsyncStoragePodCanBeRun(
String pvcStrategy, String defaultNamespaceName, int runtimesPerUser) {
return COMMON_STRATEGY.equals(pvcStrategy)
&& runtimesPerUser == 1
&& !isNullOrEmpty(defaultNamespaceName)
&& defaultNamespaceName.contains("<username>");
}
@ScheduleDelay(
unit = MINUTES,
initialDelay = 1,
delayParameterName = "che.infra.kubernetes.async.storage.shutdown_check_period_min")
public void check() {
if (isAsyncStoragePodCanBeRun
&& shutdownTimeoutSec
> 0) { // if system not support async storage mode or idling time set to 0 or less
// do nothing
for (User user :
iterateLazily((maxItems, skipCount) -> userManager.getAll(maxItems, skipCount))) {
try {
String owner = user.getId();
Map<String, String> preferences = preferenceManager.find(owner);
String lastTimeAccess = preferences.get(LAST_ACTIVITY_TIME);
String namespace = preferences.get(LAST_ACTIVE_INFRASTRUCTURE_NAMESPACE);
if (isNullOrEmpty(namespace)
|| isNullOrEmpty(lastTimeAccess)
|| !runtimes.getInProgress(owner).isEmpty()) {
continue;
}
long lastTimeAccessSec = parseLong(lastTimeAccess);
Instant expectedShutdownAfter =
ofEpochSecond(lastTimeAccessSec).plusSeconds(shutdownTimeoutSec);
if (now().isAfter(expectedShutdownAfter)) {
removeAsyncStoragePodWithoutDeployment(namespace);
RollableScalableResource<Deployment> doneableResource =
kubernetesClientFactory
.create()
.apps()
.deployments()
.inNamespace(namespace)
.withName(ASYNC_STORAGE);
if (doneableResource.get() != null) {
doneableResource.delete();
}
}
} catch (InfrastructureException | ServerException e) {
LOG.error(e.getMessage(), e);
}
}
}
}
/**
* Cleanup existed Async Storage pods running without Deployment see
* https://github.com/eclipse/che/issues/17616. Method can be removed in 7.20.x
*
* @param namespace
* @throws InfrastructureException
*/
private void removeAsyncStoragePodWithoutDeployment(String namespace)
throws InfrastructureException {
PodResource<Pod> doneablePodResource =
kubernetesClientFactory.create().pods().inNamespace(namespace).withName(ASYNC_STORAGE);
if (doneablePodResource.get() != null) {
doneablePodResource.delete();
}
}
}

402
infrastructures/kubernetes/src/main/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/AsyncStorageProvisioner.java
View File

@ -1,402 +0,0 @@
/*
* Copyright (c) 2012-2022 Red Hat, Inc.
* This program and the accompanying materials are made
* available under the terms of the Eclipse Public License 2.0
* which is available at https://www.eclipse.org/legal/epl-2.0/
*
* SPDX-License-Identifier: EPL-2.0
*
* Contributors:
* Red Hat, Inc. - initial API and implementation
*/
package org.eclipse.che.workspace.infrastructure.kubernetes.provision;
import static com.google.common.collect.ImmutableMap.of;
import static java.lang.Boolean.parseBoolean;
import static java.lang.String.format;
import static java.lang.String.valueOf;
import static java.util.Collections.singletonList;
import static org.eclipse.che.api.workspace.shared.Constants.ASYNC_PERSIST_ATTRIBUTE;
import static org.eclipse.che.api.workspace.shared.Constants.PERSIST_VOLUMES_ATTRIBUTE;
import static org.eclipse.che.workspace.infrastructure.kubernetes.Constants.CHE_DEPLOYMENT_NAME_LABEL;
import static org.eclipse.che.workspace.infrastructure.kubernetes.Constants.CHE_USER_ID_LABEL;
import static org.eclipse.che.workspace.infrastructure.kubernetes.Warnings.NOT_ABLE_TO_PROVISION_SSH_KEYS;
import static org.eclipse.che.workspace.infrastructure.kubernetes.Warnings.NOT_ABLE_TO_PROVISION_SSH_KEYS_MESSAGE;
import static org.eclipse.che.workspace.infrastructure.kubernetes.namespace.pvc.CommonPVCStrategy.COMMON_STRATEGY;
import static org.eclipse.che.workspace.infrastructure.kubernetes.namespace.pvc.EphemeralWorkspaceUtility.isEphemeral;
import io.fabric8.kubernetes.api.model.ConfigMap;
import io.fabric8.kubernetes.api.model.ConfigMapBuilder;
import io.fabric8.kubernetes.api.model.ConfigMapVolumeSourceBuilder;
import io.fabric8.kubernetes.api.model.Container;
import io.fabric8.kubernetes.api.model.ContainerBuilder;
import io.fabric8.kubernetes.api.model.ContainerPortBuilder;
import io.fabric8.kubernetes.api.model.IntOrString;
import io.fabric8.kubernetes.api.model.IntOrStringBuilder;
import io.fabric8.kubernetes.api.model.ObjectMeta;
import io.fabric8.kubernetes.api.model.ObjectMetaBuilder;
import io.fabric8.kubernetes.api.model.PersistentVolumeClaim;
import io.fabric8.kubernetes.api.model.PersistentVolumeClaimVolumeSourceBuilder;
import io.fabric8.kubernetes.api.model.PodSpec;
import io.fabric8.kubernetes.api.model.PodSpecBuilder;
import io.fabric8.kubernetes.api.model.Quantity;
import io.fabric8.kubernetes.api.model.Service;
import io.fabric8.kubernetes.api.model.ServicePort;
import io.fabric8.kubernetes.api.model.ServicePortBuilder;
import io.fabric8.kubernetes.api.model.ServiceSpec;
import io.fabric8.kubernetes.api.model.Volume;
import io.fabric8.kubernetes.api.model.VolumeBuilder;
import io.fabric8.kubernetes.api.model.VolumeMount;
import io.fabric8.kubernetes.api.model.VolumeMountBuilder;
import io.fabric8.kubernetes.api.model.apps.Deployment;
import io.fabric8.kubernetes.api.model.apps.DeploymentBuilder;
import io.fabric8.kubernetes.client.KubernetesClient;
import io.fabric8.kubernetes.client.dsl.Resource;
import io.fabric8.kubernetes.client.dsl.RollableScalableResource;
import io.fabric8.kubernetes.client.dsl.ServiceResource;
import java.util.List;
import java.util.Map;
import javax.inject.Inject;
import javax.inject.Named;
import javax.inject.Singleton;
import org.eclipse.che.api.core.ConflictException;
import org.eclipse.che.api.core.ServerException;
import org.eclipse.che.api.core.model.workspace.runtime.RuntimeIdentity;
import org.eclipse.che.api.ssh.server.SshManager;
import org.eclipse.che.api.ssh.server.model.impl.SshPairImpl;
import org.eclipse.che.api.ssh.shared.model.SshPair;
import org.eclipse.che.api.workspace.server.model.impl.WarningImpl;
import org.eclipse.che.api.workspace.server.spi.InfrastructureException;
import org.eclipse.che.workspace.infrastructure.kubernetes.KubernetesClientFactory;
import org.eclipse.che.workspace.infrastructure.kubernetes.Names;
import org.eclipse.che.workspace.infrastructure.kubernetes.environment.KubernetesEnvironment;
import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.KubernetesObjectUtil;
import org.eclipse.che.workspace.infrastructure.kubernetes.server.ServerServiceBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Configure environment for Async Storage feature (details described in issue
* https://github.com/eclipse/che/issues/15384) This environment will allow backup on workspace stop
* event and restore on restart created earlier. <br>
* Will apply only in case workspace has attributes: asyncPersist: true - persistVolumes:
* false.</br> In case workspace has attributes: asyncPersist: true - persistVolumes: true will
* throw exception.</br> Feature enabled only for 'common' PVC strategy, in other cases will throw
* exception.</br> During provision will be created: - storage Pod - service for rsync connection
* via SSH - configmap, with public part of SSH key - PVC for storing backups;
*/
@Singleton
public class AsyncStorageProvisioner {
private static final int SERVICE_PORT = 2222;
/**
* The authorized_keys file in SSH specifies the SSH keys that can be used for logging into the
* user account for which the file is configured.
*/
private static final String AUTHORIZED_KEYS = "authorized_keys";
/**
* Name of the asynchronous storage Pod and Service. Rsync command will use this Service name for
* communications: e.g.: rsync ${RSYNC_OPTIONS} --rsh="ssh ${SSH_OPTIONS}" async-storage:/{PATH}
*/
static final String ASYNC_STORAGE = "async-storage";
/** The name suffix for ConfigMap with SSH configuration */
static final String ASYNC_STORAGE_CONFIG = "async-storage-config";
/** The path of mount storage volume for file persist */
private static final String ASYNC_STORAGE_DATA_PATH = "/" + ASYNC_STORAGE;
/** The path to the authorized_keys */
private static final String SSH_KEY_PATH = "/.ssh/" + AUTHORIZED_KEYS;
/** The name of SSH key pair for rsync */
static final String SSH_KEY_NAME = "rsync-via-ssh";
/** The name of volume for mounting configuration map and authorized_keys */
private static final String CONFIG_MAP_VOLUME_NAME = "async-storage-configvolume";
/** */
private static final String STORAGE_VOLUME = "async-storage-data";
private static final Logger LOG = LoggerFactory.getLogger(AsyncStorageProvisioner.class);
private final String sidecarImagePullPolicy;
private final String pvcQuantity;
private final String asyncStorageImage;
private final String pvcAccessMode;
private final String pvcStrategy;
private final String pvcName;
private final String pvcStorageClassName;
private final SshManager sshManager;
private final KubernetesClientFactory kubernetesClientFactory;
@Inject
public AsyncStorageProvisioner(
@Named("che.workspace.sidecar.image_pull_policy") String sidecarImagePullPolicy,
@Named("che.infra.kubernetes.async.storage.image") String asyncStorageImage,
SshManager sshManager,
KubernetesClientFactory kubernetesClientFactory) {
this.sidecarImagePullPolicy = sidecarImagePullPolicy;
this.pvcQuantity = "test";
this.asyncStorageImage = asyncStorageImage;
this.pvcAccessMode = "TEST";
this.pvcStrategy = "TEST";
this.pvcName = "TEST";
this.pvcStorageClassName = "TEST";
this.sshManager = sshManager;
this.kubernetesClientFactory = kubernetesClientFactory;
}
public void provision(KubernetesEnvironment k8sEnv, RuntimeIdentity identity)
throws InfrastructureException {
if (!parseBoolean(k8sEnv.getAttributes().get(ASYNC_PERSIST_ATTRIBUTE))) {
return;
}
if (!COMMON_STRATEGY.equals(pvcStrategy)) {
String message =
format(
"Workspace configuration not valid: Asynchronous storage available only for 'common' PVC strategy, but got %s",
pvcStrategy);
LOG.warn(message);
k8sEnv.addWarning(new WarningImpl(4200, message));
throw new InfrastructureException(message);
}
if (!isEphemeral(k8sEnv.getAttributes())) {
String message =
format(
"Workspace configuration not valid: Asynchronous storage available only if '%s' attribute set to false",
PERSIST_VOLUMES_ATTRIBUTE);
LOG.warn(message);
k8sEnv.addWarning(new WarningImpl(4200, message));
throw new InfrastructureException(message);
}
String namespace = identity.getInfrastructureNamespace();
String userId = identity.getOwnerId();
KubernetesClient k8sClient = kubernetesClientFactory.create(identity.getWorkspaceId());
String configMapName = namespace + ASYNC_STORAGE_CONFIG;
createPvcIfNotExist(k8sClient, namespace, userId);
createConfigMapIfNotExist(k8sClient, namespace, configMapName, userId, k8sEnv);
createAsyncStoragePodIfNotExist(k8sClient, namespace, configMapName, userId);
createStorageServiceIfNotExist(k8sClient, namespace, userId);
}
private void createPvcIfNotExist(KubernetesClient k8sClient, String namespace, String userId) {
Resource<PersistentVolumeClaim> claimResource =
k8sClient.persistentVolumeClaims().inNamespace(namespace).withName(pvcName);
if (claimResource.get() != null) {
return; // pvc already exist
}
PersistentVolumeClaim pvc =
KubernetesObjectUtil.newPVC(pvcName, pvcAccessMode, pvcQuantity, pvcStorageClassName);
KubernetesObjectUtil.putLabel(pvc.getMetadata(), CHE_USER_ID_LABEL, userId);
k8sClient.persistentVolumeClaims().inNamespace(namespace).create(pvc);
}
/** Get or create new pair of SSH keys, this is need for securing rsync connection */
private List<SshPairImpl> getOrCreateSshPairs(String userId, KubernetesEnvironment k8sEnv)
throws InfrastructureException {
List<SshPairImpl> sshPairs;
try {
sshPairs = sshManager.getPairs(userId, "internal");
} catch (ServerException e) {
String message = format("Unable to get SSH Keys. Cause: %s", e.getMessage());
LOG.warn(message);
k8sEnv.addWarning(
new WarningImpl(
NOT_ABLE_TO_PROVISION_SSH_KEYS,
format(NOT_ABLE_TO_PROVISION_SSH_KEYS_MESSAGE, message)));
throw new InfrastructureException(e);
}
if (sshPairs.isEmpty()) {
try {
sshPairs = singletonList(sshManager.generatePair(userId, "internal", SSH_KEY_NAME));
} catch (ServerException | ConflictException e) {
String message =
format(
"Unable to generate the SSH key for async storage service. Cause: %S",
e.getMessage());
LOG.warn(message);
k8sEnv.addWarning(
new WarningImpl(
NOT_ABLE_TO_PROVISION_SSH_KEYS,
format(NOT_ABLE_TO_PROVISION_SSH_KEYS_MESSAGE, message)));
throw new InfrastructureException(e);
}
}
return sshPairs;
}
/** Create configmap with public part of SSH key */
private void createConfigMapIfNotExist(
KubernetesClient k8sClient,
String namespace,
String configMapName,
String userId,
KubernetesEnvironment k8sEnv)
throws InfrastructureException {
Resource<ConfigMap> mapResource =
k8sClient.configMaps().inNamespace(namespace).withName(configMapName);
if (mapResource.get() != null) { // map already exist
return;
}
List<SshPairImpl> sshPairs = getOrCreateSshPairs(userId, k8sEnv);
if (sshPairs == null) {
return;
}
SshPair sshPair = sshPairs.get(0);
Map<String, String> sshConfigData = of(AUTHORIZED_KEYS, sshPair.getPublicKey() + "\n");
ConfigMap configMap =
new ConfigMapBuilder()
.withNewMetadata()
.withName(configMapName)
.withNamespace(namespace)
.withLabels(of(CHE_USER_ID_LABEL, userId))
.endMetadata()
.withData(sshConfigData)
.build();
k8sClient.configMaps().inNamespace(namespace).create(configMap);
}
/**
* Create storage Pod with container with mounted volume for storing project source backups, SSH
* key and exposed port for rsync connection
*/
private void createAsyncStoragePodIfNotExist(
KubernetesClient k8sClient, String namespace, String configMap, String userId) {
RollableScalableResource<Deployment> resource =
k8sClient.apps().deployments().inNamespace(namespace).withName(ASYNC_STORAGE);
if (resource.get() != null) {
return; // deployment already exist
}
String containerName = Names.generateName(ASYNC_STORAGE);
Volume storageVolume =
new VolumeBuilder()
.withName(STORAGE_VOLUME)
.withPersistentVolumeClaim(
new PersistentVolumeClaimVolumeSourceBuilder()
.withClaimName(pvcName)
.withReadOnly(false)
.build())
.build();
Volume sshKeyVolume =
new VolumeBuilder()
.withName(CONFIG_MAP_VOLUME_NAME)
.withConfigMap(
new ConfigMapVolumeSourceBuilder()
.withName(configMap)
.withDefaultMode(0600)
.build())
.build();
VolumeMount storageVolumeMount =
new VolumeMountBuilder()
.withMountPath(ASYNC_STORAGE_DATA_PATH)
.withName(STORAGE_VOLUME)
.withReadOnly(false)
.build();
VolumeMount sshVolumeMount =
new VolumeMountBuilder()
.withMountPath(SSH_KEY_PATH)
.withSubPath(AUTHORIZED_KEYS)
.withName(CONFIG_MAP_VOLUME_NAME)
.withReadOnly(true)
.build();
Container container =
new ContainerBuilder()
.withName(containerName)
.withImage(asyncStorageImage)
.withImagePullPolicy(sidecarImagePullPolicy)
.withNewResources()
.addToLimits("memory", new Quantity("512Mi"))
.addToRequests("memory", new Quantity("256Mi"))
.endResources()
.withPorts(
new ContainerPortBuilder()
.withContainerPort(SERVICE_PORT)
.withProtocol("TCP")
.build())
.withVolumeMounts(storageVolumeMount, sshVolumeMount)
.build();
PodSpecBuilder podSpecBuilder = new PodSpecBuilder();
PodSpec podSpec =
podSpecBuilder.withContainers(container).withVolumes(storageVolume, sshKeyVolume).build();
ObjectMetaBuilder metaBuilder = new ObjectMetaBuilder();
ObjectMeta meta =
metaBuilder
.withLabels(
of(
"app",
"che",
CHE_USER_ID_LABEL,
userId,
CHE_DEPLOYMENT_NAME_LABEL,
ASYNC_STORAGE))
.withNamespace(namespace)
.withName(ASYNC_STORAGE)
.build();
Deployment deployment =
new DeploymentBuilder()
.withMetadata(meta)
.withNewSpec()
.withNewSelector()
.withMatchLabels(meta.getLabels())
.endSelector()
.withReplicas(1)
.withNewTemplate()
.withMetadata(meta)
.withSpec(podSpec)
.endTemplate()
.endSpec()
.build();
k8sClient.apps().deployments().inNamespace(namespace).create(deployment);
}
/** Create service for serving rsync connection */
private void createStorageServiceIfNotExist(
KubernetesClient k8sClient, String namespace, String userId) {
ServiceResource<Service> serviceResource =
k8sClient.services().inNamespace(namespace).withName(ASYNC_STORAGE);
if (serviceResource.get() != null) {
return; // service already exist
}
ObjectMeta meta = new ObjectMeta();
meta.setName(ASYNC_STORAGE);
meta.setNamespace(namespace);
meta.setLabels(of(CHE_USER_ID_LABEL, userId));
IntOrString targetPort =
new IntOrStringBuilder().withIntVal(SERVICE_PORT).withStrVal(valueOf(SERVICE_PORT)).build();
ServicePort port =
new ServicePortBuilder()
.withName("rsync-port")
.withProtocol("TCP")
.withPort(SERVICE_PORT)
.withTargetPort(targetPort)
.build();
ServiceSpec spec = new ServiceSpec();
spec.setPorts(singletonList(port));
spec.setSelector(of(CHE_DEPLOYMENT_NAME_LABEL, ASYNC_STORAGE));
ServerServiceBuilder serviceBuilder = new ServerServiceBuilder();
Service service =
serviceBuilder
.withPorts(singletonList(port))
.withSelectorEntry(CHE_DEPLOYMENT_NAME_LABEL, ASYNC_STORAGE)
.withName(ASYNC_STORAGE)
.build();
k8sClient.services().inNamespace(namespace).create(service);
}
}

98
infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/AsyncStorageModeValidatorTest.java
View File

@ -1,98 +0,0 @@
/*
* Copyright (c) 2012-2022 Red Hat, Inc.
* This program and the accompanying materials are made
* available under the terms of the Eclipse Public License 2.0
* which is available at https://www.eclipse.org/legal/epl-2.0/
*
* SPDX-License-Identifier: EPL-2.0
*
* Contributors:
* Red Hat, Inc. - initial API and implementation
*/
package org.eclipse.che.workspace.infrastructure.kubernetes;
import static com.google.common.collect.ImmutableMap.of;
import static org.eclipse.che.api.workspace.shared.Constants.ASYNC_PERSIST_ATTRIBUTE;
import static org.eclipse.che.api.workspace.shared.Constants.PERSIST_VOLUMES_ATTRIBUTE;
import org.eclipse.che.api.core.ValidationException;
import org.testng.annotations.Test;
public class AsyncStorageModeValidatorTest {
@Test
public void shouldBeFineForEphemeralMode() throws ValidationException {
AsyncStorageModeValidator validator = new AsyncStorageModeValidator("<username>-che", 1);
validator.validate(of(PERSIST_VOLUMES_ATTRIBUTE, "false"));
}
@Test
public void shouldBeFineForPersistentMode() throws ValidationException {
AsyncStorageModeValidator validator = new AsyncStorageModeValidator("<username>-che", 1);
validator.validate(of(PERSIST_VOLUMES_ATTRIBUTE, "true"));
}
@Test
public void shouldBeFineForEmptyAttribute() throws ValidationException {
AsyncStorageModeValidator validator = new AsyncStorageModeValidator("<username>-che", 1);
validator.validate(of());
}
@Test(
expectedExceptions = ValidationException.class,
expectedExceptionsMessageRegExp =
"Workspace configuration not valid: Asynchronous storage available only for NOT persistent storage")
public void shouldThrowExceptionIfAsyncAttributeForNotEphemeral() throws ValidationException {
AsyncStorageModeValidator validator = new AsyncStorageModeValidator("<username>-che", 1);
validator.validate(of(ASYNC_PERSIST_ATTRIBUTE, "true", PERSIST_VOLUMES_ATTRIBUTE, "true"));
}
@Test
public void shouldBeFineForEphemeralModeUpdate() throws ValidationException {
AsyncStorageModeValidator validator = new AsyncStorageModeValidator("<username>-che", 1);
validator.validateUpdate(of(), of(PERSIST_VOLUMES_ATTRIBUTE, "false"));
}
@Test
public void shouldBeFineForPersistentModeUpdate() throws ValidationException {
AsyncStorageModeValidator validator = new AsyncStorageModeValidator("<username>-che", 1);
validator.validateUpdate(of(), of(PERSIST_VOLUMES_ATTRIBUTE, "true"));
}
@Test
public void shouldBeFineForEmptyAttributeUpdate() throws ValidationException {
AsyncStorageModeValidator validator = new AsyncStorageModeValidator("<username>-che", 1);
validator.validateUpdate(of(), of());
}
@Test(
expectedExceptions = ValidationException.class,
expectedExceptionsMessageRegExp =
"Workspace configuration not valid: Asynchronous storage available only for NOT persistent storage")
public void shouldThrowExceptionIfAsyncAttributeForNotEphemeralUpdate()
throws ValidationException {
AsyncStorageModeValidator validator = new AsyncStorageModeValidator("<username>-che", 1);
validator.validateUpdate(
of(), of(ASYNC_PERSIST_ATTRIBUTE, "true", PERSIST_VOLUMES_ATTRIBUTE, "true"));
}
@Test(
expectedExceptions = ValidationException.class,
expectedExceptionsMessageRegExp =
"Workspace configuration not valid: Asynchronous storage available only for NOT persistent storage")
public void shouldThrowExceptionIfAsyncAttributeForNotEphemeralUpdate2()
throws ValidationException {
AsyncStorageModeValidator validator = new AsyncStorageModeValidator("<username>-che", 1);
validator.validateUpdate(
of(PERSIST_VOLUMES_ATTRIBUTE, "true"), of(ASYNC_PERSIST_ATTRIBUTE, "true"));
}
}

8
infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/KubernetesEnvironmentProvisionerTest.java
View File

@ -18,8 +18,6 @@ import static org.mockito.Mockito.when;
import org.eclipse.che.api.core.model.workspace.runtime.RuntimeIdentity;
import org.eclipse.che.workspace.infrastructure.kubernetes.KubernetesEnvironmentProvisioner.KubernetesEnvironmentProvisionerImpl;
import org.eclipse.che.workspace.infrastructure.kubernetes.environment.KubernetesEnvironment;
import org.eclipse.che.workspace.infrastructure.kubernetes.provision.AsyncStoragePodInterceptor;
import org.eclipse.che.workspace.infrastructure.kubernetes.provision.AsyncStorageProvisioner;
import org.eclipse.che.workspace.infrastructure.kubernetes.provision.CertificateProvisioner;
import org.eclipse.che.workspace.infrastructure.kubernetes.provision.GatewayRouterProvisioner;
import org.eclipse.che.workspace.infrastructure.kubernetes.provision.GitConfigProvisioner;
@ -75,8 +73,6 @@ public class KubernetesEnvironmentProvisionerTest {
@Mock private ImagePullSecretProvisioner imagePullSecretProvisioner;
@Mock private ProxySettingsProvisioner proxySettingsProvisioner;
@Mock private ServiceAccountProvisioner serviceAccountProvisioner;
@Mock private AsyncStorageProvisioner asyncStorageProvisioner;
@Mock private AsyncStoragePodInterceptor asyncStoragePodObserver;
@Mock private CertificateProvisioner certificateProvisioner;
@Mock private SshKeysProvisioner sshKeysProvisioner;
@Mock private GitConfigProvisioner gitConfigProvisioner;
@ -102,7 +98,6 @@ public class KubernetesEnvironmentProvisionerTest {
envVarsProvisioner,
restartPolicyRewriter,
ramLimitProvisioner,
logsVolumeMachineProvisioner,
securityContextProvisioner,
podTerminationGracePeriodProvisioner,
externalServerIngressTlsProvisionerProvider,
@ -110,8 +105,6 @@ public class KubernetesEnvironmentProvisionerTest {
proxySettingsProvisioner,
nodeSelectorProvisioner,
tolerationsProvisioner,
asyncStorageProvisioner,
asyncStoragePodObserver,
serviceAccountProvisioner,
certificateProvisioner,
sshKeysProvisioner,
@ -122,7 +115,6 @@ public class KubernetesEnvironmentProvisionerTest {
trustedCAProvisioner);
provisionOrder =
inOrder(
logsVolumeMachineProvisioner,
uniqueNamesProvisioner,
serversProvisioner,
envVarsProvisioner,

161
infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/pvc/EphemeralWorkspaceAdapterTest.java
View File

@ -1,161 +0,0 @@
/*
* Copyright (c) 2012-2021 Red Hat, Inc.
* This program and the accompanying materials are made
* available under the terms of the Eclipse Public License 2.0
* which is available at https://www.eclipse.org/legal/epl-2.0/
*
* SPDX-License-Identifier: EPL-2.0
*
* Contributors:
* Red Hat, Inc. - initial API and implementation
*/
package org.eclipse.che.workspace.infrastructure.kubernetes.namespace.pvc;
import static org.eclipse.che.api.workspace.shared.Constants.PERSIST_VOLUMES_ATTRIBUTE;
import static org.mockito.Mockito.inOrder;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertFalse;
import static org.testng.Assert.assertNull;
import static org.testng.Assert.assertTrue;
import com.google.common.collect.ImmutableMap;
import io.fabric8.kubernetes.api.model.EmptyDirVolumeSource;
import io.fabric8.kubernetes.api.model.PersistentVolumeClaim;
import io.fabric8.kubernetes.api.model.Pod;
import io.fabric8.kubernetes.api.model.PodBuilder;
import io.fabric8.kubernetes.api.model.VolumeBuilder;
import java.util.Collections;
import java.util.Map;
import org.eclipse.che.api.core.model.workspace.Workspace;
import org.eclipse.che.api.core.model.workspace.WorkspaceConfig;
import org.eclipse.che.api.core.model.workspace.runtime.RuntimeIdentity;
import org.eclipse.che.workspace.infrastructure.kubernetes.environment.KubernetesEnvironment;
import org.mockito.ArgumentCaptor;
import org.mockito.Captor;
import org.mockito.InOrder;
import org.mockito.Mock;
import org.mockito.testng.MockitoTestNGListener;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Listeners;
import org.testng.annotations.Test;
/**
* Tests {@link EphemeralWorkspaceAdapter}.
*
* @author Ilya Buziuk
* @author Angel Misevski
*/
@Listeners(MockitoTestNGListener.class)
public class EphemeralWorkspaceAdapterTest {
private static final String EPHEMERAL_WORKSPACE_ID = "workspace123";
private static final String NON_EPHEMERAL_WORKSPACE_ID = "workspace234";
private static final String POD_NAME = "pod1";
@Mock private Workspace nonEphemeralWorkspace;
@Mock private Workspace ephemeralWorkspace;
@Mock private PVCProvisioner pvcProvisioner;
@Mock private SubPathPrefixes subPathPrefixes;
private KubernetesEnvironment k8sEnv;
@Mock private RuntimeIdentity identity;
private InOrder provisionOrder;
@Captor private ArgumentCaptor<KubernetesEnvironment> k8sEnvCaptor;
private EphemeralWorkspaceAdapter ephemeralWorkspaceAdapter;
@BeforeMethod
public void setup() throws Exception {
ephemeralWorkspaceAdapter = new EphemeralWorkspaceAdapter(pvcProvisioner, subPathPrefixes);
// ephemeral workspace configuration
lenient().when(ephemeralWorkspace.getId()).thenReturn(EPHEMERAL_WORKSPACE_ID);
WorkspaceConfig ephemeralWorkspaceConfig = mock(WorkspaceConfig.class);
lenient().when(ephemeralWorkspace.getConfig()).thenReturn(ephemeralWorkspaceConfig);
Map<String, String> ephemeralConfigAttributes =
Collections.singletonMap(PERSIST_VOLUMES_ATTRIBUTE, "false");
lenient().when(ephemeralWorkspaceConfig.getAttributes()).thenReturn(ephemeralConfigAttributes);
// regular / non-ephemeral workspace configuration
lenient().when(nonEphemeralWorkspace.getId()).thenReturn(NON_EPHEMERAL_WORKSPACE_ID);
WorkspaceConfig nonEphemeralWorkspaceConfig = mock(WorkspaceConfig.class);
lenient().when(nonEphemeralWorkspace.getConfig()).thenReturn(nonEphemeralWorkspaceConfig);
Map<String, String> nonEphemeralConfigAttributes = Collections.emptyMap();
lenient().when(nonEphemeralWorkspace.getAttributes()).thenReturn(nonEphemeralConfigAttributes);
k8sEnv = KubernetesEnvironment.builder().build();
provisionOrder = inOrder(pvcProvisioner, subPathPrefixes);
}
@Test
public void testIsEphemeralWorkspace() throws Exception {
assertTrue(EphemeralWorkspaceUtility.isEphemeral(ephemeralWorkspace));
assertFalse(EphemeralWorkspaceUtility.isEphemeral(nonEphemeralWorkspace));
}
@Test
public void testProvisioningAllPVCsInWorkspace() throws Exception {
// given
PersistentVolumeClaim pvc1 = UniqueWorkspacePVCStrategyTest.newPVC("pvc1");
PersistentVolumeClaim pvc2 = UniqueWorkspacePVCStrategyTest.newPVC("pvc2");
k8sEnv.getPersistentVolumeClaims().put("pvc1", pvc1);
k8sEnv.getPersistentVolumeClaims().put("pvc2", pvc2);
when(identity.getWorkspaceId()).thenReturn(EPHEMERAL_WORKSPACE_ID);
// when
ephemeralWorkspaceAdapter.provision(k8sEnv, identity);
// then
provisionOrder
.verify(pvcProvisioner)
.provision(k8sEnv, ImmutableMap.of("pvc1", pvc1, "pvc2", pvc2));
provisionOrder.verify(pvcProvisioner).convertCheVolumes(k8sEnv, EPHEMERAL_WORKSPACE_ID);
provisionOrder
.verify(subPathPrefixes)
.prefixVolumeMountsSubpaths(k8sEnv, EPHEMERAL_WORKSPACE_ID);
}
@Test
public void testConvertsAllPVCsToEmptyDir() throws Exception {
// given
k8sEnv.getPersistentVolumeClaims().put("pvc1", mock(PersistentVolumeClaim.class));
k8sEnv.getPersistentVolumeClaims().put("pvc2", mock(PersistentVolumeClaim.class));
io.fabric8.kubernetes.api.model.Volume configMapVolume =
new VolumeBuilder().withNewConfigMap().withName("configMap").endConfigMap().build();
io.fabric8.kubernetes.api.model.Volume emptyDirVolume =
new VolumeBuilder().withNewEmptyDir().endEmptyDir().build();
io.fabric8.kubernetes.api.model.Volume pvcVolume =
new VolumeBuilder()
.withNewPersistentVolumeClaim()
.withClaimName("pvc1")
.endPersistentVolumeClaim()
.build();
Pod pod =
new PodBuilder()
.withNewMetadata()
.withName(POD_NAME)
.endMetadata()
.withNewSpec()
.withVolumes(
new VolumeBuilder(pvcVolume).build(),
new VolumeBuilder(configMapVolume).build(),
new VolumeBuilder(emptyDirVolume).build())
.endSpec()
.build();
k8sEnv.addPod(pod);
ephemeralWorkspaceAdapter.provision(k8sEnv, identity);
assertTrue(k8sEnv.getPersistentVolumeClaims().isEmpty());
assertNull(pod.getSpec().getVolumes().get(0).getPersistentVolumeClaim());
assertEquals(pod.getSpec().getVolumes().get(0).getEmptyDir(), new EmptyDirVolumeSource());
assertEquals(pod.getSpec().getVolumes().get(1), configMapVolume);
assertEquals(pod.getSpec().getVolumes().get(2), emptyDirVolume);
}
}

245
infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/namespace/pvc/UniqueWorkspacePVCStrategyTest.java
View File

@ -1,245 +0,0 @@
/*
* Copyright (c) 2012-2022 Red Hat, Inc.
* This program and the accompanying materials are made
* available under the terms of the Eclipse Public License 2.0
* which is available at https://www.eclipse.org/legal/epl-2.0/
*
* SPDX-License-Identifier: EPL-2.0
*
* Contributors:
* Red Hat, Inc. - initial API and implementation
*/
package org.eclipse.che.workspace.infrastructure.kubernetes.namespace.pvc;
import static java.util.Collections.emptyMap;
import static java.util.Collections.singletonList;
import static java.util.Collections.singletonMap;
import static org.eclipse.che.api.workspace.shared.Constants.PERSIST_VOLUMES_ATTRIBUTE;
import static org.eclipse.che.workspace.infrastructure.kubernetes.Constants.CHE_WORKSPACE_ID_LABEL;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.inOrder;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertNotNull;
import com.google.common.collect.ImmutableMap;
import io.fabric8.kubernetes.api.model.ObjectMetaBuilder;
import io.fabric8.kubernetes.api.model.PersistentVolumeClaim;
import io.fabric8.kubernetes.api.model.PersistentVolumeClaimBuilder;
import java.util.HashMap;
import java.util.Map;
import org.eclipse.che.api.core.model.workspace.Workspace;
import org.eclipse.che.api.core.model.workspace.WorkspaceConfig;
import org.eclipse.che.api.core.model.workspace.runtime.RuntimeIdentity;
import org.eclipse.che.api.workspace.server.model.impl.RuntimeIdentityImpl;
import org.eclipse.che.api.workspace.server.spi.InfrastructureException;
import org.eclipse.che.workspace.infrastructure.kubernetes.environment.KubernetesEnvironment;
import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.KubernetesNamespace;
import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.KubernetesNamespaceFactory;
import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.KubernetesPersistentVolumeClaims;
import org.mockito.ArgumentCaptor;
import org.mockito.Captor;
import org.mockito.InOrder;
import org.mockito.Mock;
import org.mockito.testng.MockitoTestNGListener;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Listeners;
import org.testng.annotations.Test;
/**
* Tests {@link UniqueWorkspacePVCStrategy}.
*
* @author Anton Korneta
* @author Sergii Leshchenko
*/
@Listeners(MockitoTestNGListener.class)
public class UniqueWorkspacePVCStrategyTest {
private static final String WORKSPACE_ID = "workspace123";
private static final String NAMESPACE = "infraNamespace";
private static final String PVC_NAME_PREFIX = "che-claim";
private static final RuntimeIdentity IDENTITY =
new RuntimeIdentityImpl(WORKSPACE_ID, "env1", "id1", NAMESPACE);
private KubernetesEnvironment k8sEnv;
@Mock private KubernetesNamespaceFactory factory;
@Mock private KubernetesNamespace k8sNamespace;
@Mock private KubernetesPersistentVolumeClaims pvcs;
@Mock private EphemeralWorkspaceAdapter ephemeralWorkspaceAdapter;
@Mock private PVCProvisioner pvcProvisioner;
@Mock private PodsVolumes podsVolumes;
@Mock private SubPathPrefixes subpathPrefixes;
@Captor private ArgumentCaptor<KubernetesEnvironment> k8sEnvCaptor;
private InOrder provisionOrder;
private UniqueWorkspacePVCStrategy strategy;
@BeforeMethod
public void setup() throws Exception {
strategy =
new UniqueWorkspacePVCStrategy(
factory, ephemeralWorkspaceAdapter, pvcProvisioner, subpathPrefixes);
k8sEnv = KubernetesEnvironment.builder().build();
provisionOrder = inOrder(pvcProvisioner, subpathPrefixes, podsVolumes);
lenient().when(factory.getOrCreate(eq(IDENTITY))).thenReturn(k8sNamespace);
lenient().when(factory.get(any(Workspace.class))).thenReturn(k8sNamespace);
lenient().when(k8sNamespace.persistentVolumeClaims()).thenReturn(pvcs);
}
@Test
public void testProvisionVolumesIntoKubernetesEnvironment() throws Exception {
// given
PersistentVolumeClaim pvc1 = newPVC("pvc1");
PersistentVolumeClaim pvc2 = newPVC("pvc2");
k8sEnv.getPersistentVolumeClaims().put("pvc1", pvc1);
k8sEnv.getPersistentVolumeClaims().put("pvc2", pvc2);
PersistentVolumeClaim existingPVC = newPVC("existingPVC");
when(pvcs.getByLabel(CHE_WORKSPACE_ID_LABEL, WORKSPACE_ID))
.thenReturn(singletonList(existingPVC));
// when
strategy.provision(k8sEnv, IDENTITY);
// then
provisionOrder
.verify(pvcProvisioner)
.provision(k8sEnvCaptor.capture(), eq(ImmutableMap.of("pvc1", pvc1, "pvc2", pvc2)));
provisionOrder.verify(pvcProvisioner).convertCheVolumes(k8sEnv, WORKSPACE_ID);
provisionOrder.verify(subpathPrefixes).prefixVolumeMountsSubpaths(k8sEnv, WORKSPACE_ID);
assertEquals(k8sEnv.getPersistentVolumeClaims().size(), 1);
assertNotNull(k8sEnv.getPersistentVolumeClaims().get("existingPVC"));
;
}
@Test
public void shouldProvisionWorkspaceIdLabelToPVCs() throws Exception {
// given
PersistentVolumeClaim existingPVC = newPVC("existingPVC");
when(pvcs.getByLabel(CHE_WORKSPACE_ID_LABEL, WORKSPACE_ID))
.thenReturn(singletonList(existingPVC));
// when
strategy.provision(k8sEnv, IDENTITY);
// then
assertEquals(k8sEnv.getPersistentVolumeClaims().size(), 1);
PersistentVolumeClaim pvc = k8sEnv.getPersistentVolumeClaims().get("existingPVC");
assertNotNull(pvc);
assertEquals(pvc.getMetadata().getLabels().get(CHE_WORKSPACE_ID_LABEL), WORKSPACE_ID);
}
@Test
public void testCreatesProvisionedPVCsOnPrepare() throws Exception {
final String uniqueName = PVC_NAME_PREFIX + "-3121";
final PersistentVolumeClaim pvc = newPVC(uniqueName);
k8sEnv.getPersistentVolumeClaims().clear();
k8sEnv.getPersistentVolumeClaims().putAll(singletonMap(uniqueName, pvc));
strategy.prepare(k8sEnv, IDENTITY, 100, emptyMap());
verify(pvcs).createIfNotExist(any());
verify(pvcs).waitBound(uniqueName, 100);
}
@Test(expectedExceptions = InfrastructureException.class)
public void throwsInfrastructureExceptionWhenFailedToCreatePVCs() throws Exception {
final PersistentVolumeClaim pvc = mock(PersistentVolumeClaim.class);
when(pvc.getMetadata()).thenReturn(new ObjectMetaBuilder().withName(PVC_NAME_PREFIX).build());
k8sEnv.getPersistentVolumeClaims().clear();
k8sEnv.getPersistentVolumeClaims().put(PVC_NAME_PREFIX, pvc);
doThrow(InfrastructureException.class).when(pvcs).createIfNotExist(any());
strategy.prepare(k8sEnv, IDENTITY, 100, emptyMap());
}
@Test
public void shouldDeletePVCsIfThereIsNoPersistAttributeInWorkspaceConfigWhenCleanupCalled()
throws Exception {
// given
Workspace workspace = mock(Workspace.class);
lenient().when(workspace.getId()).thenReturn(WORKSPACE_ID);
WorkspaceConfig workspaceConfig = mock(WorkspaceConfig.class);
lenient().when(workspace.getConfig()).thenReturn(workspaceConfig);
Map<String, String> workspaceConfigAttributes = new HashMap<>();
lenient().when(workspaceConfig.getAttributes()).thenReturn(workspaceConfigAttributes);
// when
strategy.cleanup(workspace);
// then
verify(pvcs).delete(ImmutableMap.of(CHE_WORKSPACE_ID_LABEL, WORKSPACE_ID));
}
@Test
public void shouldDeletePVCsIfPersistAttributeIsSetToTrueInWorkspaceConfigWhenCleanupCalled()
throws Exception {
// given
Workspace workspace = mock(Workspace.class);
lenient().when(workspace.getId()).thenReturn(WORKSPACE_ID);
WorkspaceConfig workspaceConfig = mock(WorkspaceConfig.class);
lenient().when(workspace.getConfig()).thenReturn(workspaceConfig);
Map<String, String> workspaceConfigAttributes = new HashMap<>();
lenient().when(workspaceConfig.getAttributes()).thenReturn(workspaceConfigAttributes);
workspaceConfigAttributes.put(PERSIST_VOLUMES_ATTRIBUTE, "true");
// when
strategy.cleanup(workspace);
// then
verify(pvcs).delete(ImmutableMap.of(CHE_WORKSPACE_ID_LABEL, WORKSPACE_ID));
}
@Test
public void shouldDoNothingIfPersistAttributeIsSetToFalseInWorkspaceConfigWhenCleanupCalled()
throws Exception {
// given
Workspace workspace = mock(Workspace.class);
lenient().when(workspace.getId()).thenReturn(WORKSPACE_ID);
WorkspaceConfig workspaceConfig = mock(WorkspaceConfig.class);
lenient().when(workspace.getConfig()).thenReturn(workspaceConfig);
Map<String, String> workspaceConfigAttributes = new HashMap<>();
lenient().when(workspaceConfig.getAttributes()).thenReturn(workspaceConfigAttributes);
workspaceConfigAttributes.put(PERSIST_VOLUMES_ATTRIBUTE, "false");
// when
strategy.cleanup(workspace);
// then
verify(pvcs, never()).delete(ImmutableMap.of(CHE_WORKSPACE_ID_LABEL, WORKSPACE_ID));
}
static PersistentVolumeClaim newPVC(String name) {
return newPVC(name, new HashMap<>());
}
static PersistentVolumeClaim newPVC(String name, Map<String, String> labels) {
return new PersistentVolumeClaimBuilder()
.withNewMetadata()
.withName(name)
.withLabels(labels)
.endMetadata()
.withNewSpec()
.endSpec()
.build();
}
}

69
infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/AsyncStoragePodInterceptorTest.java
View File

@ -1,69 +0,0 @@
/*
* Copyright (c) 2012-2022 Red Hat, Inc.
* This program and the accompanying materials are made
* available under the terms of the Eclipse Public License 2.0
* which is available at https://www.eclipse.org/legal/epl-2.0/
*
* SPDX-License-Identifier: EPL-2.0
*
* Contributors:
* Red Hat, Inc. - initial API and implementation
*/
package org.eclipse.che.workspace.infrastructure.kubernetes.provision;
import static org.mockito.Mockito.verifyNoMoreInteractions;
import io.fabric8.kubernetes.api.model.Pod;
import io.fabric8.kubernetes.api.model.apps.Deployment;
import io.fabric8.kubernetes.client.KubernetesClient;
import io.fabric8.kubernetes.client.dsl.AppsAPIGroupDSL;
import io.fabric8.kubernetes.client.dsl.EditReplacePatchDeletable;
import io.fabric8.kubernetes.client.dsl.MixedOperation;
import io.fabric8.kubernetes.client.dsl.NonNamespaceOperation;
import io.fabric8.kubernetes.client.dsl.PodResource;
import io.fabric8.kubernetes.client.dsl.RollableScalableResource;
import java.util.UUID;
import org.eclipse.che.api.core.model.workspace.runtime.RuntimeIdentity;
import org.eclipse.che.workspace.infrastructure.kubernetes.KubernetesClientFactory;
import org.eclipse.che.workspace.infrastructure.kubernetes.environment.KubernetesEnvironment;
import org.mockito.Mock;
import org.mockito.testng.MockitoTestNGListener;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Listeners;
import org.testng.annotations.Test;
@Listeners(MockitoTestNGListener.class)
public class AsyncStoragePodInterceptorTest {
private static final String WORKSPACE_ID = UUID.randomUUID().toString();
private static final String NAMESPACE = UUID.randomUUID().toString();
@Mock private KubernetesEnvironment kubernetesEnvironment;
@Mock private RuntimeIdentity identity;
@Mock private KubernetesClientFactory clientFactory;
@Mock private KubernetesClient kubernetesClient;
@Mock private RollableScalableResource<Deployment> deploymentResource;
@Mock private PodResource<Pod> podResource;
@Mock private MixedOperation mixedOperation;
@Mock private MixedOperation mixedOperationPod;
@Mock private NonNamespaceOperation namespaceOperation;
@Mock private NonNamespaceOperation namespacePodOperation;
@Mock private EditReplacePatchDeletable<Deployment> deletable;
@Mock private AppsAPIGroupDSL apps;
private AsyncStoragePodInterceptor asyncStoragePodInterceptor;
@BeforeMethod
public void setUp() {
asyncStoragePodInterceptor = new AsyncStoragePodInterceptor(clientFactory);
}
@Test
public void shouldDoNothingIfNotCommonStrategy() throws Exception {
AsyncStoragePodInterceptor asyncStoragePodInterceptor =
new AsyncStoragePodInterceptor(clientFactory);
asyncStoragePodInterceptor.intercept(kubernetesEnvironment, identity);
verifyNoMoreInteractions(clientFactory);
verifyNoMoreInteractions(identity);
}
}

157
infrastructures/kubernetes/src/test/java/org/eclipse/che/workspace/infrastructure/kubernetes/provision/AsyncStoragePodWatcherTest.java
View File

@ -1,157 +0,0 @@
/*
* Copyright (c) 2012-2022 Red Hat, Inc.
* This program and the accompanying materials are made
* available under the terms of the Eclipse Public License 2.0
* which is available at https://www.eclipse.org/legal/epl-2.0/
*
* SPDX-License-Identifier: EPL-2.0
*
* Contributors:
* Red Hat, Inc. - initial API and implementation
*/
package org.eclipse.che.workspace.infrastructure.kubernetes.provision;
import static java.time.Instant.now;
import static java.util.UUID.randomUUID;
import static org.eclipse.che.workspace.infrastructure.kubernetes.provision.AsyncStorageProvisioner.ASYNC_STORAGE;
import static org.mockito.ArgumentMatchers.anyInt;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.verifyNoMoreInteractions;
import io.fabric8.kubernetes.api.model.Pod;
import io.fabric8.kubernetes.api.model.apps.Deployment;
import io.fabric8.kubernetes.client.KubernetesClient;
import io.fabric8.kubernetes.client.dsl.AppsAPIGroupDSL;
import io.fabric8.kubernetes.client.dsl.MixedOperation;
import io.fabric8.kubernetes.client.dsl.NonNamespaceOperation;
import io.fabric8.kubernetes.client.dsl.PodResource;
import io.fabric8.kubernetes.client.dsl.RollableScalableResource;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import org.eclipse.che.api.core.Page;
import org.eclipse.che.api.user.server.PreferenceManager;
import org.eclipse.che.api.user.server.UserManager;
import org.eclipse.che.api.user.server.model.impl.UserImpl;
import org.eclipse.che.api.workspace.server.WorkspaceRuntimes;
import org.eclipse.che.api.workspace.shared.Constants;
import org.eclipse.che.workspace.infrastructure.kubernetes.KubernetesClientFactory;
import org.mockito.Mock;
import org.mockito.testng.MockitoTestNGListener;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Listeners;
import org.testng.annotations.Test;
@Listeners(value = {MockitoTestNGListener.class})
public class AsyncStoragePodWatcherTest {
private final String NAMESPACE = randomUUID().toString();
private final String WORKSPACE_ID = randomUUID().toString();
private final String USER_ID = randomUUID().toString();
private Map<String, String> userPref;
@Mock private KubernetesClientFactory kubernetesClientFactory;
@Mock private UserManager userManager;
@Mock private PreferenceManager preferenceManager;
@Mock private WorkspaceRuntimes runtimes;
@Mock private KubernetesClient kubernetesClient;
@Mock private RollableScalableResource<Deployment> deploymentResource;
@Mock private MixedOperation mixedOperation;
@Mock private NonNamespaceOperation namespaceOperation;
@Mock private PodResource<Pod> podResource;
@Mock private MixedOperation mixedOperationPod;
@Mock private NonNamespaceOperation namespacePodOperation;
@Mock private UserImpl user;
@Mock private AppsAPIGroupDSL apps;
@BeforeMethod
public void setUp() throws Exception {
lenient().when(user.getId()).thenReturn(USER_ID);
userPref = new HashMap<>(3);
long epochSecond = now().getEpochSecond();
long activityTime = epochSecond - 600; // stored time 10 minutes early
userPref.put(Constants.LAST_ACTIVITY_TIME, Long.toString(activityTime));
userPref.put(Constants.LAST_ACTIVE_INFRASTRUCTURE_NAMESPACE, NAMESPACE);
lenient().when(preferenceManager.find(USER_ID)).thenReturn(userPref);
Page<UserImpl> userPage = new Page<>(Collections.singleton(user), 0, 1, 1);
lenient().when(userManager.getAll(anyInt(), anyLong())).thenReturn(userPage);
lenient().when(kubernetesClientFactory.create()).thenReturn(kubernetesClient);
lenient().when(kubernetesClient.apps()).thenReturn(apps);
lenient().when(apps.deployments()).thenReturn(mixedOperation);
lenient().when(mixedOperation.inNamespace(NAMESPACE)).thenReturn(namespaceOperation);
lenient().when(namespaceOperation.withName(ASYNC_STORAGE)).thenReturn(deploymentResource);
lenient().when(kubernetesClient.pods()).thenReturn(mixedOperationPod);
lenient().when(mixedOperationPod.inNamespace(NAMESPACE)).thenReturn(namespacePodOperation);
lenient().when(namespacePodOperation.withName(ASYNC_STORAGE)).thenReturn(podResource);
lenient().when(podResource.get()).thenReturn(null);
}
@Test
public void shouldDoNothingIfNotCommonPvcStrategy() throws Exception {
AsyncStoragePodWatcher watcher =
new AsyncStoragePodWatcher(
kubernetesClientFactory, userManager, preferenceManager, runtimes, 1, "<username>", 1);
watcher.check();
verifyNoMoreInteractions(preferenceManager);
verifyNoMoreInteractions(kubernetesClientFactory);
verifyNoMoreInteractions(deploymentResource);
}
@Test
public void shouldDoNothingIfAllowedUserDefinedNamespaces() throws Exception {
AsyncStoragePodWatcher watcher =
new AsyncStoragePodWatcher(
kubernetesClientFactory, userManager, preferenceManager, runtimes, 1, "<username>", 1);
watcher.check();
verifyNoMoreInteractions(preferenceManager);
verifyNoMoreInteractions(kubernetesClientFactory);
verifyNoMoreInteractions(deploymentResource);
}
@Test
public void shouldDoNothingIfDefaultNamespaceNotCorrect() throws Exception {
AsyncStoragePodWatcher watcher =
new AsyncStoragePodWatcher(
kubernetesClientFactory, userManager, preferenceManager, runtimes, 1, "<foo-bar>", 1);
watcher.check();
verifyNoMoreInteractions(preferenceManager);
verifyNoMoreInteractions(kubernetesClientFactory);
verifyNoMoreInteractions(deploymentResource);
}
@Test
public void shouldDoNothingIfAllowMoreThanOneRuntime() throws Exception {
AsyncStoragePodWatcher watcher =
new AsyncStoragePodWatcher(
kubernetesClientFactory, userManager, preferenceManager, runtimes, 1, "<foo-bar>", 2);
watcher.check();
verifyNoMoreInteractions(preferenceManager);
verifyNoMoreInteractions(kubernetesClientFactory);
verifyNoMoreInteractions(deploymentResource);
}
@Test
public void shouldDoNothingIfShutdownTimeSetToZero() throws Exception {
AsyncStoragePodWatcher watcher =
new AsyncStoragePodWatcher(
kubernetesClientFactory, userManager, preferenceManager, runtimes, 0, "<username>", 1);
watcher.check();
verifyNoMoreInteractions(preferenceManager);
verifyNoMoreInteractions(kubernetesClientFactory);
verifyNoMoreInteractions(deploymentResource);
}
}

11
infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftEnvironmentProvisioner.java
View File

@ -19,8 +19,6 @@ import org.eclipse.che.commons.annotation.Traced;
import org.eclipse.che.commons.tracing.TracingTags;
import org.eclipse.che.workspace.infrastructure.kubernetes.KubernetesEnvironmentProvisioner;
import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.pvc.WorkspaceVolumesStrategy;
import org.eclipse.che.workspace.infrastructure.kubernetes.provision.AsyncStoragePodInterceptor;
import org.eclipse.che.workspace.infrastructure.kubernetes.provision.AsyncStorageProvisioner;
import org.eclipse.che.workspace.infrastructure.kubernetes.provision.CertificateProvisioner;
import org.eclipse.che.workspace.infrastructure.kubernetes.provision.DeploymentMetadataProvisioner;
import org.eclipse.che.workspace.infrastructure.kubernetes.provision.GatewayRouterProvisioner;
@ -76,9 +74,7 @@ public class OpenShiftEnvironmentProvisioner
private final ProxySettingsProvisioner proxySettingsProvisioner;
private final NodeSelectorProvisioner nodeSelectorProvisioner;
private final TolerationsProvisioner tolerationsProvisioner;
private final AsyncStoragePodInterceptor asyncStoragePodInterceptor;
private final ServiceAccountProvisioner serviceAccountProvisioner;
private final AsyncStorageProvisioner asyncStorageProvisioner;
private final CertificateProvisioner certificateProvisioner;
private final SshKeysProvisioner sshKeysProvisioner;
private final GitConfigProvisioner gitConfigProvisioner;
@ -103,8 +99,6 @@ public class OpenShiftEnvironmentProvisioner
ProxySettingsProvisioner proxySettingsProvisioner,
NodeSelectorProvisioner nodeSelectorProvisioner,
TolerationsProvisioner tolerationsProvisioner,
AsyncStorageProvisioner asyncStorageProvisioner,
AsyncStoragePodInterceptor asyncStoragePodInterceptor,
ServiceAccountProvisioner serviceAccountProvisioner,
CertificateProvisioner certificateProvisioner,
SshKeysProvisioner sshKeysProvisioner,
@ -128,8 +122,6 @@ public class OpenShiftEnvironmentProvisioner
this.proxySettingsProvisioner = proxySettingsProvisioner;
this.nodeSelectorProvisioner = nodeSelectorProvisioner;
this.tolerationsProvisioner = tolerationsProvisioner;
this.asyncStorageProvisioner = asyncStorageProvisioner;
this.asyncStoragePodInterceptor = asyncStoragePodInterceptor;
this.serviceAccountProvisioner = serviceAccountProvisioner;
this.certificateProvisioner = certificateProvisioner;
this.sshKeysProvisioner = sshKeysProvisioner;
@ -151,7 +143,7 @@ public class OpenShiftEnvironmentProvisioner
"Start provisioning OpenShift environment for workspace '{}'", identity.getWorkspaceId());
// 1 stage - update environment according Infrastructure specific
if (pvcEnabled) {
asyncStoragePodInterceptor.intercept(osEnv, identity);
// TODO: Remove things related to pvcEnabled boolean
logsVolumeMachineProvisioner.provision(osEnv, identity);
}
@ -173,7 +165,6 @@ public class OpenShiftEnvironmentProvisioner
imagePullSecretProvisioner.provision(osEnv, identity);
proxySettingsProvisioner.provision(osEnv, identity);
serviceAccountProvisioner.provision(osEnv, identity);
asyncStorageProvisioner.provision(osEnv, identity);
certificateProvisioner.provision(osEnv, identity);
sshKeysProvisioner.provision(osEnv, identity);
vcsSslCertificateProvisioner.provision(osEnv, identity);

8
infrastructures/openshift/src/main/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftInfraModule.java
View File

@ -34,7 +34,6 @@ import org.eclipse.che.api.workspace.server.spi.provision.env.CheApiInternalEnvV
import org.eclipse.che.api.workspace.server.spi.provision.env.EnvVarProvider;
import org.eclipse.che.api.workspace.server.wsplugins.ChePluginsApplier;
import org.eclipse.che.api.workspace.shared.Constants;
import org.eclipse.che.workspace.infrastructure.kubernetes.AsyncStorageModeValidator;
import org.eclipse.che.workspace.infrastructure.kubernetes.InconsistentRuntimesDetector;
import org.eclipse.che.workspace.infrastructure.kubernetes.K8sInfraNamespaceWsAttributeValidator;
import org.eclipse.che.workspace.infrastructure.kubernetes.KubernetesClientFactory;
@ -58,9 +57,6 @@ import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.configurato
import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.configurator.UserPreferencesConfigurator;
import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.configurator.UserProfileConfigurator;
import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.pvc.WorkspacePVCCleaner;
import org.eclipse.che.workspace.infrastructure.kubernetes.provision.AsyncStoragePodInterceptor;
import org.eclipse.che.workspace.infrastructure.kubernetes.provision.AsyncStoragePodWatcher;
import org.eclipse.che.workspace.infrastructure.kubernetes.provision.AsyncStorageProvisioner;
import org.eclipse.che.workspace.infrastructure.kubernetes.provision.GatewayTlsProvisioner;
import org.eclipse.che.workspace.infrastructure.kubernetes.provision.KubernetesCheApiExternalEnvVarProvider;
import org.eclipse.che.workspace.infrastructure.kubernetes.provision.KubernetesCheApiInternalEnvVarProvider;
@ -109,7 +105,6 @@ public class OpenShiftInfraModule extends AbstractModule {
Multibinder<WorkspaceAttributeValidator> workspaceAttributeValidators =
Multibinder.newSetBinder(binder(), WorkspaceAttributeValidator.class);
workspaceAttributeValidators.addBinding().to(K8sInfraNamespaceWsAttributeValidator.class);
workspaceAttributeValidators.addBinding().to(AsyncStorageModeValidator.class);
Multibinder<NamespaceConfigurator> namespaceConfigurators =
Multibinder.newSetBinder(binder(), NamespaceConfigurator.class);
@ -251,8 +246,5 @@ public class OpenShiftInfraModule extends AbstractModule {
bind(ExternalServiceExposureStrategy.class).toProvider(ServiceExposureStrategyProvider.class);
bind(CookiePathStrategy.class).to(OpenShiftCookiePathStrategy.class);
bind(NonTlsDistributedClusterModeNotifier.class);
bind(AsyncStorageProvisioner.class);
bind(AsyncStoragePodInterceptor.class);
bind(AsyncStoragePodWatcher.class);
}
}

6
infrastructures/openshift/src/test/java/org/eclipse/che/workspace/infrastructure/openshift/OpenShiftEnvironmentProvisionerTest.java
View File

@ -17,8 +17,6 @@ import static org.mockito.Mockito.when;
import org.eclipse.che.api.core.model.workspace.runtime.RuntimeIdentity;
import org.eclipse.che.workspace.infrastructure.kubernetes.namespace.pvc.WorkspaceVolumesStrategy;
import org.eclipse.che.workspace.infrastructure.kubernetes.provision.AsyncStoragePodInterceptor;
import org.eclipse.che.workspace.infrastructure.kubernetes.provision.AsyncStorageProvisioner;
import org.eclipse.che.workspace.infrastructure.kubernetes.provision.CertificateProvisioner;
import org.eclipse.che.workspace.infrastructure.kubernetes.provision.DeploymentMetadataProvisioner;
import org.eclipse.che.workspace.infrastructure.kubernetes.provision.GatewayRouterProvisioner;
@ -72,8 +70,6 @@ public class OpenShiftEnvironmentProvisionerTest {
@Mock private ImagePullSecretProvisioner imagePullSecretProvisioner;
@Mock private ProxySettingsProvisioner proxySettingsProvisioner;
@Mock private ServiceAccountProvisioner serviceAccountProvisioner;
@Mock private AsyncStorageProvisioner asyncStorageProvisioner;
@Mock private AsyncStoragePodInterceptor asyncStoragePodObserver;
@Mock private CertificateProvisioner certificateProvisioner;
@Mock private SshKeysProvisioner sshKeysProvisioner;
@Mock private GitConfigProvisioner gitConfigProvisioner;
@ -107,8 +103,6 @@ public class OpenShiftEnvironmentProvisionerTest {
proxySettingsProvisioner,
nodeSelectorProvisioner,
tolerationsProvisioner,
asyncStorageProvisioner,
asyncStoragePodObserver,
serviceAccountProvisioner,
certificateProvisioner,
sshKeysProvisioner,