Factories for private Gitlab repository with pre-created personal access token (#19351)

7.30.x
Max Shaposhnik 2021-03-30 14:19:23 +03:00 committed by GitHub
parent 97824c857a
commit e99fcd78b9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 249 additions and 155 deletions

View File

@ -11,101 +11,23 @@
*/
package org.eclipse.che.api.factory.server.bitbucket;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.Optional;
import org.eclipse.che.api.factory.server.scm.AuthorizingFileContentProvider;
import org.eclipse.che.api.factory.server.scm.GitCredentialManager;
import org.eclipse.che.api.factory.server.scm.PersonalAccessToken;
import org.eclipse.che.api.factory.server.scm.PersonalAccessTokenManager;
import org.eclipse.che.api.factory.server.scm.exception.ScmCommunicationException;
import org.eclipse.che.api.factory.server.scm.exception.ScmConfigurationPersistenceException;
import org.eclipse.che.api.factory.server.scm.exception.ScmUnauthorizedException;
import org.eclipse.che.api.factory.server.scm.exception.UnknownScmProviderException;
import org.eclipse.che.api.factory.server.scm.exception.UnsatisfiedScmPreconditionException;
import org.eclipse.che.api.workspace.server.devfile.FileContentProvider;
import org.eclipse.che.api.workspace.server.devfile.URLFetcher;
import org.eclipse.che.api.workspace.server.devfile.exception.DevfileException;
import org.eclipse.che.commons.env.EnvironmentContext;
/**
* Bitbucket specific file content provider. Files are retrieved using bitbucket REST API and
* personal access token based authentication is performed during requests.
*/
public class BitbucketServerAuthorizingFileContentProvider implements FileContentProvider {
private final URLFetcher urlFetcher;
private final BitbucketUrl bitbucketUrl;
private final GitCredentialManager gitCredentialManager;
private final PersonalAccessTokenManager personalAccessTokenManager;
public class BitbucketServerAuthorizingFileContentProvider
extends AuthorizingFileContentProvider<BitbucketUrl> {
public BitbucketServerAuthorizingFileContentProvider(
BitbucketUrl bitbucketUrl,
URLFetcher urlFetcher,
GitCredentialManager gitCredentialManager,
PersonalAccessTokenManager personalAccessTokenManager) {
this.bitbucketUrl = bitbucketUrl;
this.urlFetcher = urlFetcher;
this.gitCredentialManager = gitCredentialManager;
this.personalAccessTokenManager = personalAccessTokenManager;
}
@Override
public String fetchContent(String fileURL) throws IOException, DevfileException {
String requestURL;
try {
if (new URI(fileURL).isAbsolute()) {
requestURL = fileURL;
} else {
// since files retrieved via REST, we cannot use path symbols like . ./ so cut them off
requestURL = bitbucketUrl.rawFileLocation(fileURL.replaceAll("^[/.]+", ""));
}
} catch (URISyntaxException e) {
throw new DevfileException(e.getMessage(), e);
}
try {
Optional<PersonalAccessToken> token =
personalAccessTokenManager.get(
EnvironmentContext.getCurrent().getSubject(), bitbucketUrl.getHostName());
if (token.isPresent()) {
PersonalAccessToken personalAccessToken = token.get();
String content = urlFetcher.fetch(requestURL, "Bearer " + personalAccessToken.getToken());
gitCredentialManager.createOrReplace(personalAccessToken);
return content;
} else {
try {
return urlFetcher.fetch(requestURL);
} catch (IOException exception) {
// unable to determine exact cause, so let's just try to authorize...
try {
PersonalAccessToken personalAccessToken =
personalAccessTokenManager.fetchAndSave(
EnvironmentContext.getCurrent().getSubject(), bitbucketUrl.getHostName());
String content =
urlFetcher.fetch(requestURL, "Bearer " + personalAccessToken.getToken());
gitCredentialManager.createOrReplace(personalAccessToken);
return content;
} catch (ScmUnauthorizedException
| ScmCommunicationException
| UnknownScmProviderException e) {
throw new DevfileException(e.getMessage(), e);
}
}
}
} catch (IOException e) {
throw new IOException(
String.format(
"Failed to fetch a content from URL %s. Make sure the URL"
+ " is correct. Additionally, if you're using "
+ " relative form, make sure the referenced files are actually stored"
+ " relative to the devfile on the same host,"
+ " or try to specify URL in absolute form. The current attempt to download"
+ " the file failed with the following error message: %s",
fileURL, e.getMessage()),
e);
} catch (ScmConfigurationPersistenceException | UnsatisfiedScmPreconditionException e) {
throw new DevfileException(e.getMessage(), e);
}
super(bitbucketUrl, urlFetcher, personalAccessTokenManager, gitCredentialManager);
}
}

View File

@ -31,6 +31,8 @@ import org.eclipse.che.api.factory.server.urlfactory.RemoteFactoryUrl;
*/
public class GithubUrl implements RemoteFactoryUrl {
private static final String HOSTNAME = "https://github.com/";
/** Username part of github URL */
private String username;
@ -160,12 +162,17 @@ public class GithubUrl implements RemoteFactoryUrl {
.toString();
}
@Override
public String getHostName() {
return HOSTNAME;
}
/**
* Provides location to the repository part of the full github URL.
*
* @return location of the repository.
*/
protected String repositoryLocation() {
return "https://github.com/" + this.username + "/" + this.repository;
return HOSTNAME + this.username + "/" + this.repository;
}
}

View File

@ -0,0 +1,29 @@
/*
* Copyright (c) 2012-2018 Red Hat, Inc.
* This program and the accompanying materials are made
* available under the terms of the Eclipse Public License 2.0
* which is available at https://www.eclipse.org/legal/epl-2.0/
*
* SPDX-License-Identifier: EPL-2.0
*
* Contributors:
* Red Hat, Inc. - initial API and implementation
*/
package org.eclipse.che.api.factory.server.gitlab;
import org.eclipse.che.api.factory.server.scm.AuthorizingFileContentProvider;
import org.eclipse.che.api.factory.server.scm.GitCredentialManager;
import org.eclipse.che.api.factory.server.scm.PersonalAccessTokenManager;
import org.eclipse.che.api.workspace.server.devfile.URLFetcher;
/** Gitlab specific authorizing file content provider. */
class GitlabAuthorizingFileContentProvider extends AuthorizingFileContentProvider<GitlabUrl> {
GitlabAuthorizingFileContentProvider(
GitlabUrl githubUrl,
URLFetcher urlFetcher,
GitCredentialManager gitCredentialManager,
PersonalAccessTokenManager personalAccessTokenManager) {
super(githubUrl, urlFetcher, personalAccessTokenManager, gitCredentialManager);
}
}

View File

@ -22,6 +22,8 @@ import javax.validation.constraints.NotNull;
import org.eclipse.che.api.core.ApiException;
import org.eclipse.che.api.core.BadRequestException;
import org.eclipse.che.api.factory.server.DefaultFactoryParameterResolver;
import org.eclipse.che.api.factory.server.scm.GitCredentialManager;
import org.eclipse.che.api.factory.server.scm.PersonalAccessTokenManager;
import org.eclipse.che.api.factory.server.urlfactory.URLFactoryBuilder;
import org.eclipse.che.api.factory.shared.dto.FactoryDto;
import org.eclipse.che.api.factory.shared.dto.FactoryMetaDto;
@ -40,11 +42,20 @@ public class GitlabFactoryParametersResolver extends DefaultFactoryParameterReso
private final GitlabUrlParser gitlabURLParser;
private final GitCredentialManager gitCredentialManager;
private final PersonalAccessTokenManager personalAccessTokenManager;
@Inject
public GitlabFactoryParametersResolver(
URLFactoryBuilder urlFactoryBuilder, URLFetcher urlFetcher, GitlabUrlParser gitlabURLParser) {
URLFactoryBuilder urlFactoryBuilder,
URLFetcher urlFetcher,
GitlabUrlParser gitlabURLParser,
GitCredentialManager gitCredentialManager,
PersonalAccessTokenManager personalAccessTokenManager) {
super(urlFactoryBuilder, urlFetcher);
this.gitlabURLParser = gitlabURLParser;
this.gitCredentialManager = gitCredentialManager;
this.personalAccessTokenManager = personalAccessTokenManager;
}
/**
@ -76,7 +87,8 @@ public class GitlabFactoryParametersResolver extends DefaultFactoryParameterReso
return urlFactoryBuilder
.createFactoryFromDevfile(
gitlabUrl,
new GitlabFileContentProvider(gitlabUrl, urlFetcher),
new GitlabAuthorizingFileContentProvider(
gitlabUrl, urlFetcher, gitCredentialManager, personalAccessTokenManager),
extractOverrideParams(factoryParameters))
.orElseGet(() -> newDto(FactoryDto.class).withV(CURRENT_VERSION).withSource("repo"))
.acceptVisitor(new GitlabFactoryVisitor(gitlabUrl));

View File

@ -1,46 +0,0 @@
/*
* Copyright (c) 2012-2018 Red Hat, Inc.
* This program and the accompanying materials are made
* available under the terms of the Eclipse Public License 2.0
* which is available at https://www.eclipse.org/legal/epl-2.0/
*
* SPDX-License-Identifier: EPL-2.0
*
* Contributors:
* Red Hat, Inc. - initial API and implementation
*/
package org.eclipse.che.api.factory.server.gitlab;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import org.eclipse.che.api.workspace.server.devfile.FileContentProvider;
import org.eclipse.che.api.workspace.server.devfile.URLFetcher;
import org.eclipse.che.api.workspace.server.devfile.exception.DevfileException;
/** Gitlab specific file content provider. */
class GitlabFileContentProvider implements FileContentProvider {
private final GitlabUrl gitlabUrl;
private final URLFetcher urlFetcher;
GitlabFileContentProvider(GitlabUrl githubUrl, URLFetcher urlFetcher) {
this.gitlabUrl = githubUrl;
this.urlFetcher = urlFetcher;
}
@Override
public String fetchContent(String fileURL) throws IOException, DevfileException {
String requestURL;
try {
if (new URI(fileURL).isAbsolute()) {
requestURL = fileURL;
} else {
requestURL = gitlabUrl.rawFileLocation(fileURL);
}
} catch (URISyntaxException e) {
throw new DevfileException(e.getMessage(), e);
}
return urlFetcher.fetch(requestURL);
}
}

View File

@ -11,8 +11,9 @@
*/
package org.eclipse.che.api.factory.server.gitlab;
import static com.google.common.base.MoreObjects.firstNonNull;
import static java.net.URLEncoder.encode;
import com.google.common.base.Charsets;
import com.google.common.base.Strings;
import java.util.ArrayList;
import java.util.List;
@ -185,15 +186,23 @@ public class GitlabUrl implements RemoteFactoryUrl {
* @return location of specified file in a repository
*/
public String rawFileLocation(String fileName) {
StringJoiner joiner = new StringJoiner("/").add(hostName).add(username).add(project);
if (repository != null) {
joiner.add(repository);
String resultUrl =
new StringJoiner("/")
.add(hostName)
.add("api/v4/projects")
// use URL-encoded path to the project as a selector instead of id
.add(encode(username + "/" + project, Charsets.UTF_8))
.add("repository")
.add("files")
.add(fileName)
.add("raw")
.toString();
if (branch != null) {
resultUrl = resultUrl + "?ref=" + branch;
} else {
resultUrl = resultUrl + "?ref=master";
}
joiner.add("-").add("raw").add(firstNonNull(branch, "master"));
if (subfolder != null) {
joiner.add(subfolder);
}
return joiner.add(fileName).toString();
return resultUrl;
}
/**

View File

@ -14,12 +14,21 @@ package org.eclipse.che.api.factory.server.gitlab;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.verify;
import org.eclipse.che.api.factory.server.scm.GitCredentialManager;
import org.eclipse.che.api.factory.server.scm.PersonalAccessTokenManager;
import org.eclipse.che.api.workspace.server.devfile.FileContentProvider;
import org.eclipse.che.api.workspace.server.devfile.URLFetcher;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.testng.MockitoTestNGListener;
import org.testng.annotations.Listeners;
import org.testng.annotations.Test;
public class GitlabFileContentProviderTest {
@Listeners(MockitoTestNGListener.class)
public class GitlabAuthorizingFileContentProviderTest {
@Mock private GitCredentialManager gitCredentialsManager;
@Mock private PersonalAccessTokenManager personalAccessTokenManager;
@Test
public void shouldExpandRelativePaths() throws Exception {
@ -29,9 +38,14 @@ public class GitlabFileContentProviderTest {
.withHostName("https://gitlab.net")
.withUsername("eclipse")
.withProject("che");
FileContentProvider fileContentProvider = new GitlabFileContentProvider(gitlabUrl, urlFetcher);
FileContentProvider fileContentProvider =
new GitlabAuthorizingFileContentProvider(
gitlabUrl, urlFetcher, gitCredentialsManager, personalAccessTokenManager);
fileContentProvider.fetchContent("devfile.yaml");
verify(urlFetcher).fetch(eq("https://gitlab.net/eclipse/che/-/raw/master/devfile.yaml"));
verify(urlFetcher)
.fetch(
eq(
"https://gitlab.net/api/v4/projects/eclipse%2Fche/repository/files/devfile.yaml/raw?ref=master"));
}
@Test
@ -39,8 +53,11 @@ public class GitlabFileContentProviderTest {
URLFetcher urlFetcher = Mockito.mock(URLFetcher.class);
GitlabUrl gitlabUrl =
new GitlabUrl().withHostName("gitlab.net").withUsername("eclipse").withProject("che");
FileContentProvider fileContentProvider = new GitlabFileContentProvider(gitlabUrl, urlFetcher);
String url = "https://gitlab.net/eclipse/che/-/raw/master/devfile.yaml";
FileContentProvider fileContentProvider =
new GitlabAuthorizingFileContentProvider(
gitlabUrl, urlFetcher, gitCredentialsManager, personalAccessTokenManager);
String url =
"https://gitlab.net/api/v4/projects/eclipse%2Fche/repository/files/devfile.yaml/raw?ref=master";
fileContentProvider.fetchContent(url);
verify(urlFetcher).fetch(eq(url));
}

View File

@ -11,6 +11,7 @@
*/
package org.eclipse.che.api.factory.server.gitlab;
import static java.lang.String.format;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.when;
import static org.testng.Assert.assertEquals;
@ -57,30 +58,32 @@ public class GitlabUrlTest {
GitlabUrl gitlabUrl = gitlabUrlParser.parse(repoUrl);
assertEquals(gitlabUrl.devfileFileLocations().size(), 2);
Iterator<DevfileLocation> iterator = gitlabUrl.devfileFileLocations().iterator();
assertEquals(iterator.next().location(), fileUrl + "devfile.yaml");
assertEquals(iterator.next().location(), fileUrl + "foo.bar");
assertEquals(iterator.next().location(), format(fileUrl, "devfile.yaml"));
assertEquals(iterator.next().location(), format(fileUrl, "foo.bar"));
}
@DataProvider
public static Object[][] urlsProvider() {
return new Object[][] {
{"https://gitlab.net/eclipse/che.git", "https://gitlab.net/eclipse/che/-/raw/master/"},
{
"https://gitlab.net/eclipse/che.git",
"https://gitlab.net/api/v4/projects/eclipse%%2Fche/repository/files/%s/raw?ref=master"
},
{
"https://gitlab.net/eclipse/fooproj/che.git",
"https://gitlab.net/eclipse/fooproj/che/-/raw/master/"
"https://gitlab.net/api/v4/projects/eclipse%%2Ffooproj/repository/files/%s/raw?ref=master"
},
{
"https://gitlab.net/eclipse/fooproj/-/tree/master/",
"https://gitlab.net/eclipse/fooproj/-/raw/master/"
"https://gitlab.net/api/v4/projects/eclipse%%2Ffooproj/repository/files/%s/raw?ref=master"
},
{
"https://gitlab.net/eclipse/fooproj/che/-/tree/foobranch/",
"https://gitlab.net/eclipse/fooproj/che/-/raw/foobranch/"
"https://gitlab.net/api/v4/projects/eclipse%%2Ffooproj/repository/files/%s/raw?ref=foobranch"
},
{
"https://gitlab.net/eclipse/fooproj/che/-/tree/foobranch/subfolder",
"https://gitlab.net/eclipse/fooproj/che/-/raw/foobranch/subfolder/"
"https://gitlab.net/api/v4/projects/eclipse%%2Ffooproj/repository/files/%s/raw?ref=foobranch"
},
};
}

View File

@ -0,0 +1,109 @@
/*
* Copyright (c) 2012-2018 Red Hat, Inc.
* This program and the accompanying materials are made
* available under the terms of the Eclipse Public License 2.0
* which is available at https://www.eclipse.org/legal/epl-2.0/
*
* SPDX-License-Identifier: EPL-2.0
*
* Contributors:
* Red Hat, Inc. - initial API and implementation
*/
package org.eclipse.che.api.factory.server.scm;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.util.Optional;
import org.eclipse.che.api.factory.server.scm.exception.ScmCommunicationException;
import org.eclipse.che.api.factory.server.scm.exception.ScmConfigurationPersistenceException;
import org.eclipse.che.api.factory.server.scm.exception.ScmUnauthorizedException;
import org.eclipse.che.api.factory.server.scm.exception.UnknownScmProviderException;
import org.eclipse.che.api.factory.server.scm.exception.UnsatisfiedScmPreconditionException;
import org.eclipse.che.api.factory.server.urlfactory.RemoteFactoryUrl;
import org.eclipse.che.api.workspace.server.devfile.FileContentProvider;
import org.eclipse.che.api.workspace.server.devfile.URLFetcher;
import org.eclipse.che.api.workspace.server.devfile.exception.DevfileException;
import org.eclipse.che.commons.env.EnvironmentContext;
/**
* Common implementation of file content provider which is able to access content of private
* repositories using personal access tokens from specially formatted secret in user's namespace.
*/
public class AuthorizingFileContentProvider<T extends RemoteFactoryUrl>
implements FileContentProvider {
private final T remoteFactoryUrl;
private final URLFetcher urlFetcher;
private final PersonalAccessTokenManager personalAccessTokenManager;
private final GitCredentialManager gitCredentialManager;
public AuthorizingFileContentProvider(
T remoteFactoryUrl,
URLFetcher urlFetcher,
PersonalAccessTokenManager personalAccessTokenManager,
GitCredentialManager gitCredentialManager) {
this.remoteFactoryUrl = remoteFactoryUrl;
this.urlFetcher = urlFetcher;
this.personalAccessTokenManager = personalAccessTokenManager;
this.gitCredentialManager = gitCredentialManager;
}
@Override
public String fetchContent(String fileURL) throws IOException, DevfileException {
String requestURL;
try {
if (new URI(fileURL).isAbsolute()) {
requestURL = fileURL;
} else {
// since files retrieved via REST, we cannot use path symbols like . ./ so cut them off
requestURL = remoteFactoryUrl.rawFileLocation(fileURL.replaceAll("^[/.]+", ""));
}
} catch (URISyntaxException e) {
throw new DevfileException(e.getMessage(), e);
}
try {
Optional<PersonalAccessToken> token =
personalAccessTokenManager.get(
EnvironmentContext.getCurrent().getSubject(), remoteFactoryUrl.getHostName());
if (token.isPresent()) {
PersonalAccessToken personalAccessToken = token.get();
String content = urlFetcher.fetch(requestURL, "Bearer " + personalAccessToken.getToken());
gitCredentialManager.createOrReplace(personalAccessToken);
return content;
} else {
try {
return urlFetcher.fetch(requestURL);
} catch (IOException exception) {
// unable to determine exact cause, so let's just try to authorize...
try {
PersonalAccessToken personalAccessToken =
personalAccessTokenManager.fetchAndSave(
EnvironmentContext.getCurrent().getSubject(), remoteFactoryUrl.getHostName());
String content =
urlFetcher.fetch(requestURL, "Bearer " + personalAccessToken.getToken());
gitCredentialManager.createOrReplace(personalAccessToken);
return content;
} catch (ScmUnauthorizedException
| ScmCommunicationException
| UnknownScmProviderException e) {
throw new DevfileException(e.getMessage(), e);
}
}
}
} catch (IOException e) {
throw new IOException(
String.format(
"Failed to fetch a content from URL %s. Make sure the URL"
+ " is correct. Additionally, if you're using "
+ " relative form, make sure the referenced files are actually stored"
+ " relative to the devfile on the same host,"
+ " or try to specify URL in absolute form. The current attempt to download"
+ " the file failed with the following error message: %s",
fileURL, e.getMessage()),
e);
} catch (ScmConfigurationPersistenceException | UnsatisfiedScmPreconditionException e) {
throw new DevfileException(e.getMessage(), e);
}
}
}

View File

@ -13,6 +13,7 @@ package org.eclipse.che.api.factory.server.urlfactory;
import static java.util.Collections.singletonList;
import java.net.URI;
import java.util.List;
import java.util.Optional;
@ -40,6 +41,16 @@ public class DefaultFactoryUrl implements RemoteFactoryUrl {
});
}
@Override
public String rawFileLocation(String filename) {
return URI.create(devfileFileLocation).resolve(filename).toString();
}
@Override
public String getHostName() {
return URI.create(devfileFileLocation).getHost();
}
public DefaultFactoryUrl withDevfileFileLocation(String devfileFileLocation) {
this.devfileFileLocation = devfileFileLocation;
return this;

View File

@ -27,6 +27,12 @@ public interface RemoteFactoryUrl {
*/
List<DevfileLocation> devfileFileLocations();
/** Address of raw file content in remote repository */
String rawFileLocation(String filename);
/** Remote hostname */
String getHostName();
/** Describes devfile location, including filename if any. */
interface DevfileLocation {
Optional<String> filename();

View File

@ -34,6 +34,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode;
import java.io.IOException;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import org.eclipse.che.api.core.ApiException;
@ -173,8 +174,10 @@ public class URLFactoryBuilderTest {
when(devfileVersionDetector.devfileMajorVersion(devfile)).thenReturn(2);
RemoteFactoryUrl githubLikeRemoteUrl =
() ->
Collections.singletonList(
new RemoteFactoryUrl() {
@Override
public List<DevfileLocation> devfileFileLocations() {
return Collections.singletonList(
new DevfileLocation() {
@Override
public Optional<String> filename() {
@ -186,6 +189,18 @@ public class URLFactoryBuilderTest {
return myLocation;
}
});
}
@Override
public String rawFileLocation(String filename) {
return null;
}
@Override
public String getHostName() {
return null;
}
};
FactoryMetaDto factory =
urlFactoryBuilder