fix: Override bitbucket content provider to use API request (#399)

bitbucket.org doesn't allow to fetch raw file content from private repositories using oAuth token any more. Override the common fetch content flow specifically for bitbucket.
2022-12-09 10:46:01 +02:00 · 2022-12-09 10:46:01 +02:00 · ea76cda24c
parent 3a9a03f9d1
commit ea76cda24c
8 changed files with 158 additions and 51 deletions
--- a/wsmaster/che-core-api-factory-bitbucket/src/main/java/org/eclipse/che/api/factory/server/bitbucket/BitbucketApiClient.java
+++ b/wsmaster/che-core-api-factory-bitbucket/src/main/java/org/eclipse/che/api/factory/server/bitbucket/BitbucketApiClient.java
@ -115,6 +115,26 @@ public class BitbucketApiClient {
        });
  }

+  public String getFileContent(
+      String workspace, String repository, String source, String path, String authenticationToken)
+      throws ScmItemNotFoundException, ScmCommunicationException, ScmBadRequestException {
+    final URI uri =
+        apiServerUrl.resolve(
+            String.format("repositories/%s/%s/src/%s/%s", workspace, repository, source, path));
+    HttpRequest request = buildBitbucketApiRequest(uri, authenticationToken);
+    LOG.trace("executeRequest={}", request);
+    return executeRequest(
+        httpClient,
+        request,
+        response -> {
+          try {
+            return CharStreams.toString(new InputStreamReader(response.body(), Charsets.UTF_8));
+          } catch (IOException e) {
+            throw new UncheckedIOException(e);
+          }
+        });
+  }
+
  /**
   * Returns email of the user, associated with the provided OAuth access token.
   *
--- a/wsmaster/che-core-api-factory-bitbucket/src/main/java/org/eclipse/che/api/factory/server/bitbucket/BitbucketAuthorizingFileContentProvider.java
+++ b/wsmaster/che-core-api-factory-bitbucket/src/main/java/org/eclipse/che/api/factory/server/bitbucket/BitbucketAuthorizingFileContentProvider.java
@ -11,19 +11,33 @@
 */
 package org.eclipse.che.api.factory.server.bitbucket;

+import java.io.FileNotFoundException;
 import java.io.IOException;
 import org.eclipse.che.api.factory.server.scm.AuthorizingFileContentProvider;
+import org.eclipse.che.api.factory.server.scm.PersonalAccessToken;
 import org.eclipse.che.api.factory.server.scm.PersonalAccessTokenManager;
+import org.eclipse.che.api.factory.server.scm.exception.ScmBadRequestException;
+import org.eclipse.che.api.factory.server.scm.exception.ScmCommunicationException;
+import org.eclipse.che.api.factory.server.scm.exception.ScmConfigurationPersistenceException;
+import org.eclipse.che.api.factory.server.scm.exception.ScmItemNotFoundException;
+import org.eclipse.che.api.factory.server.scm.exception.ScmUnauthorizedException;
+import org.eclipse.che.api.factory.server.scm.exception.UnknownScmProviderException;
+import org.eclipse.che.api.factory.server.scm.exception.UnsatisfiedScmPreconditionException;
 import org.eclipse.che.api.workspace.server.devfile.URLFetcher;
+import org.eclipse.che.api.workspace.server.devfile.exception.DevfileException;

 /** Bitbucket specific authorizing file content provider. */
 class BitbucketAuthorizingFileContentProvider extends AuthorizingFileContentProvider<BitbucketUrl> {

+  private final BitbucketApiClient apiClient;
+
  BitbucketAuthorizingFileContentProvider(
      BitbucketUrl bitbucketUrl,
      URLFetcher urlFetcher,
-      PersonalAccessTokenManager personalAccessTokenManager) {
+      PersonalAccessTokenManager personalAccessTokenManager,
+      BitbucketApiClient apiClient) {
    super(bitbucketUrl, urlFetcher, personalAccessTokenManager);
+    this.apiClient = apiClient;
  }

  /** Formats OAuth token as HTTP Authorization header. */
@ -32,6 +46,34 @@ class BitbucketAuthorizingFileContentProvider extends AuthorizingFileContentProv
    return "Bearer " + token;
  }

+  @Override
+  public String fetchContent(String fileURL) throws IOException, DevfileException {
+    final String requestURL = formatUrl(fileURL);
+    try {
+      // try to authenticate for the given URL
+      PersonalAccessToken token =
+          personalAccessTokenManager.getAndStore(remoteFactoryUrl.getHostName());
+      String[] split = requestURL.split("/");
+      return apiClient.getFileContent(
+          split[3],
+          split[4],
+          split[6],
+          fileURL.substring(fileURL.indexOf(split[6]) + split[6].length() + 1),
+          token.getToken());
+    } catch (UnknownScmProviderException e) {
+      return fetchContentWithoutToken(requestURL, e);
+    } catch (ScmCommunicationException e) {
+      return toIOException(fileURL, e);
+    } catch (ScmUnauthorizedException
+        | ScmConfigurationPersistenceException
+        | UnsatisfiedScmPreconditionException
+        | ScmBadRequestException e) {
+      throw new DevfileException(e.getMessage(), e);
+    } catch (ScmItemNotFoundException e) {
+      throw new FileNotFoundException(e.getMessage());
+    }
+  }
+
  @Override
  protected boolean isPublicRepository(BitbucketUrl remoteFactoryUrl) {
    try {
--- a/wsmaster/che-core-api-factory-bitbucket/src/main/java/org/eclipse/che/api/factory/server/bitbucket/BitbucketFactoryParametersResolver.java
+++ b/wsmaster/che-core-api-factory-bitbucket/src/main/java/org/eclipse/che/api/factory/server/bitbucket/BitbucketFactoryParametersResolver.java
@ -51,6 +51,8 @@ public class BitbucketFactoryParametersResolver extends DefaultFactoryParameterR
  /** Personal Access Token manager used when fetching protected content. */
  private final PersonalAccessTokenManager personalAccessTokenManager;

+  private final BitbucketApiClient bitbucketApiClient;
+
  @Inject
  public BitbucketFactoryParametersResolver(
      BitbucketURLParser bitbucketURLParser,
@ -58,12 +60,14 @@ public class BitbucketFactoryParametersResolver extends DefaultFactoryParameterR
      BitbucketSourceStorageBuilder bitbucketSourceStorageBuilder,
      URLFactoryBuilder urlFactoryBuilder,
      ProjectConfigDtoMerger projectConfigDtoMerger,
-      PersonalAccessTokenManager personalAccessTokenManager) {
+      PersonalAccessTokenManager personalAccessTokenManager,
+      BitbucketApiClient bitbucketApiClient) {
    super(urlFactoryBuilder, urlFetcher);
    this.bitbucketURLParser = bitbucketURLParser;
    this.bitbucketSourceStorageBuilder = bitbucketSourceStorageBuilder;
    this.projectConfigDtoMerger = projectConfigDtoMerger;
    this.personalAccessTokenManager = personalAccessTokenManager;
+    this.bitbucketApiClient = bitbucketApiClient;
  }

  /**
@ -98,7 +102,7 @@ public class BitbucketFactoryParametersResolver extends DefaultFactoryParameterR
        .createFactoryFromDevfile(
            bitbucketUrl,
            new BitbucketAuthorizingFileContentProvider(
-                bitbucketUrl, urlFetcher, personalAccessTokenManager),
+                bitbucketUrl, urlFetcher, personalAccessTokenManager, bitbucketApiClient),
            extractOverrideParams(factoryParameters),
            false)
        .orElseGet(() -> newDto(FactoryDto.class).withV(CURRENT_VERSION).withSource("repo"))
--- a/wsmaster/che-core-api-factory-bitbucket/src/main/java/org/eclipse/che/api/factory/server/bitbucket/BitbucketScmFileResolver.java
+++ b/wsmaster/che-core-api-factory-bitbucket/src/main/java/org/eclipse/che/api/factory/server/bitbucket/BitbucketScmFileResolver.java
@ -29,15 +29,18 @@ public class BitbucketScmFileResolver implements ScmFileResolver {
  private final BitbucketURLParser bitbucketUrlParser;
  private final URLFetcher urlFetcher;
  private final PersonalAccessTokenManager personalAccessTokenManager;
+  private final BitbucketApiClient bitbucketApiClient;

  @Inject
  public BitbucketScmFileResolver(
      BitbucketURLParser bitbucketUrlParser,
      URLFetcher urlFetcher,
-      PersonalAccessTokenManager personalAccessTokenManager) {
+      PersonalAccessTokenManager personalAccessTokenManager,
+      BitbucketApiClient bitbucketApiClient) {
    this.bitbucketUrlParser = bitbucketUrlParser;
    this.urlFetcher = urlFetcher;
    this.personalAccessTokenManager = personalAccessTokenManager;
+    this.bitbucketApiClient = bitbucketApiClient;
  }

  @Override
@ -52,7 +55,7 @@ public class BitbucketScmFileResolver implements ScmFileResolver {
    final BitbucketUrl bitbucketUrl = bitbucketUrlParser.parse(repository);
    try {
      return new BitbucketAuthorizingFileContentProvider(
-              bitbucketUrl, urlFetcher, personalAccessTokenManager)
+              bitbucketUrl, urlFetcher, personalAccessTokenManager, bitbucketApiClient)
          .fetchContent(bitbucketUrl.rawFileLocation(filePath));
    } catch (IOException e) {
      throw new NotFoundException(e.getMessage());
--- a/wsmaster/che-core-api-factory-bitbucket/src/test/java/org/eclipse/che/api/factory/server/bitbucket/BitbucketAuthorizingFileContentProviderTest.java
+++ b/wsmaster/che-core-api-factory-bitbucket/src/test/java/org/eclipse/che/api/factory/server/bitbucket/BitbucketAuthorizingFileContentProviderTest.java
@ -14,6 +14,7 @@ package org.eclipse.che.api.factory.server.bitbucket;
 import static org.mockito.ArgumentMatchers.*;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
+import static org.testng.Assert.assertEquals;

 import java.io.FileNotFoundException;
 import org.eclipse.che.api.factory.server.scm.PersonalAccessToken;
@ -31,6 +32,7 @@ import org.testng.annotations.Test;
 public class BitbucketAuthorizingFileContentProviderTest {

  @Mock private PersonalAccessTokenManager personalAccessTokenManager;
+  @Mock private BitbucketApiClient bitbucketApiClient;

  @Test
  public void shouldExpandRelativePaths() throws Exception {
@ -38,13 +40,11 @@ public class BitbucketAuthorizingFileContentProviderTest {
    BitbucketUrl bitbucketUrl = new BitbucketUrl().withWorkspaceId("eclipse").withRepository("che");
    FileContentProvider fileContentProvider =
        new BitbucketAuthorizingFileContentProvider(
-            bitbucketUrl, urlFetcher, personalAccessTokenManager);
-    var personalAccessToken = new PersonalAccessToken("foo", "che", "my-token");
-    when(personalAccessTokenManager.getAndStore(anyString())).thenReturn(personalAccessToken);
+            bitbucketUrl, urlFetcher, personalAccessTokenManager, bitbucketApiClient);
+    when(personalAccessTokenManager.getAndStore(anyString()))
+        .thenThrow(UnknownScmProviderException.class);
    fileContentProvider.fetchContent("devfile.yaml");
-    verify(urlFetcher)
-        .fetch(
-            eq("https://bitbucket.org/eclipse/che/raw/HEAD/devfile.yaml"), eq("Bearer my-token"));
+    verify(urlFetcher).fetch(eq("https://bitbucket.org/eclipse/che/raw/HEAD/devfile.yaml"));
  }

  @Test
@ -53,12 +53,12 @@ public class BitbucketAuthorizingFileContentProviderTest {
    BitbucketUrl bitbucketUrl = new BitbucketUrl().withUsername("eclipse").withRepository("che");
    FileContentProvider fileContentProvider =
        new BitbucketAuthorizingFileContentProvider(
-            bitbucketUrl, urlFetcher, personalAccessTokenManager);
+            bitbucketUrl, urlFetcher, personalAccessTokenManager, bitbucketApiClient);
    String url = "https://api.bitbucket.org/2.0/repositories/foo/bar/devfile.yaml";
-    var personalAccessToken = new PersonalAccessToken(url, "che", "my-token");
-    when(personalAccessTokenManager.getAndStore(anyString())).thenReturn(personalAccessToken);
+    when(personalAccessTokenManager.getAndStore(anyString()))
+        .thenThrow(UnknownScmProviderException.class);
    fileContentProvider.fetchContent(url);
-    verify(urlFetcher).fetch(eq(url), eq("Bearer my-token"));
+    verify(urlFetcher).fetch(eq(url));
  }

  @Test(expectedExceptions = FileNotFoundException.class)
@ -73,7 +73,29 @@ public class BitbucketAuthorizingFileContentProviderTest {
        new BitbucketUrl().withUsername("eclipse").withWorkspaceId("eclipse").withRepository("che");
    FileContentProvider fileContentProvider =
        new BitbucketAuthorizingFileContentProvider(
-            bitbucketUrl, urlFetcher, personalAccessTokenManager);
+            bitbucketUrl, urlFetcher, personalAccessTokenManager, bitbucketApiClient);
    fileContentProvider.fetchContent(url);
  }
+
+  @Test
+  public void shouldFetchContent() throws Exception {
+    // given
+    URLFetcher urlFetcher = Mockito.mock(URLFetcher.class);
+    String url = "https://bitbucket.org/workspace/repository/raw/HEAD/devfile.yaml";
+    PersonalAccessToken personalAccessToken = new PersonalAccessToken(url, "che", "my-token");
+    when(personalAccessTokenManager.getAndStore(anyString())).thenReturn(personalAccessToken);
+    when(bitbucketApiClient.getFileContent(
+            eq("workspace"), eq("repository"), eq("HEAD"), eq("devfile.yaml"), eq("my-token")))
+        .thenReturn("content");
+    BitbucketUrl bitbucketUrl =
+        new BitbucketUrl().withUsername("eclipse").withWorkspaceId("eclipse").withRepository("che");
+    FileContentProvider fileContentProvider =
+        new BitbucketAuthorizingFileContentProvider(
+            bitbucketUrl, urlFetcher, personalAccessTokenManager, bitbucketApiClient);
+    // when
+    String content = fileContentProvider.fetchContent(url);
+
+    // then
+    assertEquals(content, "content");
+  }
 }
--- a/wsmaster/che-core-api-factory-bitbucket/src/test/java/org/eclipse/che/api/factory/server/bitbucket/BitbucketFactoryParametersResolverTest.java
+++ b/wsmaster/che-core-api-factory-bitbucket/src/test/java/org/eclipse/che/api/factory/server/bitbucket/BitbucketFactoryParametersResolverTest.java
@ -79,6 +79,7 @@ public class BitbucketFactoryParametersResolverTest {
  @Mock private URLFactoryBuilder urlFactoryBuilder;

  @Mock private PersonalAccessTokenManager personalAccessTokenManager;
+  @Mock private BitbucketApiClient bitbucketApiClient;

  /**
   * Capturing the location parameter when calling {@link
@ -100,7 +101,8 @@ public class BitbucketFactoryParametersResolverTest {
            bitbucketSourceStorageBuilder,
            urlFactoryBuilder,
            projectConfigDtoMerger,
-            personalAccessTokenManager);
+            personalAccessTokenManager,
+            bitbucketApiClient);
    assertNotNull(this.bitbucketFactoryParametersResolver);
  }

--- a/wsmaster/che-core-api-factory-bitbucket/src/test/java/org/eclipse/che/api/factory/server/bitbucket/BitbucketScmFileResolverTest.java
+++ b/wsmaster/che-core-api-factory-bitbucket/src/test/java/org/eclipse/che/api/factory/server/bitbucket/BitbucketScmFileResolverTest.java
@ -12,6 +12,7 @@
 package org.eclipse.che.api.factory.server.bitbucket;

 import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.eq;
 import static org.mockito.Mockito.when;
 import static org.testng.AssertJUnit.assertEquals;
 import static org.testng.AssertJUnit.assertFalse;
@ -37,6 +38,7 @@ public class BitbucketScmFileResolverTest {

  @Mock private DevfileFilenamesProvider devfileFilenamesProvider;
  @Mock private PersonalAccessTokenManager personalAccessTokenManager;
+  @Mock private BitbucketApiClient bitbucketApiClient;

  private BitbucketScmFileResolver bitbucketScmFileResolver;

@ -45,7 +47,8 @@ public class BitbucketScmFileResolverTest {
    bitbucketURLParser = new BitbucketURLParser(devfileFilenamesProvider);
    assertNotNull(this.bitbucketURLParser);
    bitbucketScmFileResolver =
-        new BitbucketScmFileResolver(bitbucketURLParser, urlFetcher, personalAccessTokenManager);
+        new BitbucketScmFileResolver(
+            bitbucketURLParser, urlFetcher, personalAccessTokenManager, bitbucketApiClient);
    assertNotNull(this.bitbucketScmFileResolver);
  }

@ -67,7 +70,9 @@ public class BitbucketScmFileResolverTest {
  public void shouldReturnContentFromUrlFetcher() throws Exception {
    final String rawContent = "raw_content";
    final String filename = "devfile.yaml";
-    when(urlFetcher.fetch(anyString(), anyString())).thenReturn(rawContent);
+    when(bitbucketApiClient.getFileContent(
+            eq("test"), eq("repo"), eq("HEAD"), eq("devfile.yaml"), eq("my-token")))
+        .thenReturn(rawContent);
    var personalAccessToken = new PersonalAccessToken("foo", "che", "my-token");
    when(personalAccessTokenManager.getAndStore(anyString())).thenReturn(personalAccessToken);

--- a/wsmaster/che-core-api-factory/src/main/java/org/eclipse/che/api/factory/server/scm/AuthorizingFileContentProvider.java
+++ b/wsmaster/che-core-api-factory/src/main/java/org/eclipse/che/api/factory/server/scm/AuthorizingFileContentProvider.java
@ -70,39 +70,9 @@ public class AuthorizingFileContentProvider<T extends RemoteFactoryUrl>
        return urlFetcher.fetch(requestURL, formatAuthorization(token.getToken()));
      }
    } catch (UnknownScmProviderException e) {
-      // we don't have any provider matching this SCM provider
-      // so try without secrets being configured
-      try {
-        return urlFetcher.fetch(requestURL);
-      } catch (IOException exception) {
-        if (exception instanceof SSLException) {
-          ScmCommunicationException cause =
-              new ScmCommunicationException(
-                  String.format(
-                      "Failed to fetch a content from URL %s due to TLS key misconfiguration. Please refer to the docs about how to correctly import it. ",
-                      requestURL));
-          throw new DevfileException(exception.getMessage(), cause);
-        } else if (exception instanceof FileNotFoundException) {
-          if (isPublicRepository(remoteFactoryUrl)) {
-            // for public repo-s return 404 as-is
-            throw exception;
-          }
-        }
-        throw new DevfileException(
-            String.format("%s: %s", e.getMessage(), exception.getMessage()), exception);
-      }
+      return fetchContentWithoutToken(requestURL, e);
    } catch (ScmCommunicationException e) {
-      throw new IOException(
-          String.format(
-              "Failed to fetch a content from URL %s. Make sure the URL"
-                  + " is correct. For private repository, make sure authentication is configured."
-                  + " Additionally, if you're using "
-                  + " relative form, make sure the referenced file are actually stored"
-                  + " relative to the devfile on the same host,"
-                  + " or try to specify URL in absolute form. The current attempt to authenticate"
-                  + " request, failed with the following error message: %s",
-              fileURL, e.getMessage()),
-          e);
+      return toIOException(fileURL, e);
    } catch (ScmUnauthorizedException
        | ScmConfigurationPersistenceException
        | UnsatisfiedScmPreconditionException e) {
@ -110,6 +80,45 @@ public class AuthorizingFileContentProvider<T extends RemoteFactoryUrl>
    }
  }

+  protected String fetchContentWithoutToken(String requestURL, UnknownScmProviderException e)
+      throws DevfileException, IOException {
+    // we don't have any provider matching this SCM provider
+    // so try without secrets being configured
+    try {
+      return urlFetcher.fetch(requestURL);
+    } catch (IOException exception) {
+      if (exception instanceof SSLException) {
+        ScmCommunicationException cause =
+            new ScmCommunicationException(
+                String.format(
+                    "Failed to fetch a content from URL %s due to TLS key misconfiguration. Please refer to the docs about how to correctly import it. ",
+                    requestURL));
+        throw new DevfileException(exception.getMessage(), cause);
+      } else if (exception instanceof FileNotFoundException) {
+        if (isPublicRepository(remoteFactoryUrl)) {
+          // for public repo-s return 404 as-is
+          throw exception;
+        }
+      }
+      throw new DevfileException(
+          String.format("%s: %s", e.getMessage(), exception.getMessage()), exception);
+    }
+  }
+
+  protected String toIOException(String fileURL, ScmCommunicationException e) throws IOException {
+    throw new IOException(
+        String.format(
+            "Failed to fetch a content from URL %s. Make sure the URL"
+                + " is correct. For private repository, make sure authentication is configured."
+                + " Additionally, if you're using "
+                + " relative form, make sure the referenced file are actually stored"
+                + " relative to the devfile on the same host,"
+                + " or try to specify URL in absolute form. The current attempt to authenticate"
+                + " request, failed with the following error message: %s",
+            fileURL, e.getMessage()),
+        e);
+  }
+
  protected boolean isPublicRepository(T remoteFactoryUrl) {
    return false;
  }