[ Theia Docker image] Updates (#10552)

* - move all Dockerfile logic to separate setup file
- allow to apply patches on top of existing theia instance
- recompile all extensions to ensure its compatible with current theia version
- Then we publish internally a new version on a local registry, and change dependency version on all extensions to make compliant with this version
- install globally typescript so typescript lsp is working

remaining 4 extensions not installed:
 - https://github.com/eclipse/che-theia-github-plugin
-  https://github.com/eclipse/che-theia-task-plugin
- https://github.com/eclipse/che-theia-java-plugin
 - https://github.com/eclipse/che-theia-terminal-plugin (but terminal is not compliant for openshift)

Change-Id: Ic6ba219e35b0e83783dc37b28f05dd54f2864cf2
Signed-off-by: Florent BENOIT <fbenoit@redhat.com>

dockerfiles/theia/Dockerfile

@@ -7,10 +7,10 @@
 # Contributors:
 #   Red Hat, Inc. - initial API and implementation
 
-FROM node:8-alpine
+FROM node:9-alpine
 
 ARG GITHUB_TOKEN
-ARG THEIA_VERSION=0.4.0-next.b17727c1
+ARG THEIA_VERSION=0.3.13
 
 EXPOSE 3000 3030
 
@@ -23,49 +23,7 @@ WORKDIR ${HOME}
 ADD src/ ${HOME}/
 ADD supervisord.conf /etc/
 
-RUN \
-    # Install basic software used for checking github API rate limit
-    apk update && apk add --no-cache curl jq ca-certificates openssl && \
-    # update certificates
-    update-ca-certificates && \
-    # define in env variable GITHUB_TOKEN only if it is defined
-    # else check if github rate limit is enough, else will abort requiring to set GITHUB_TOKEN value
-    if [ "$GITHUB_TOKEN" != "" ]; then \
-      export GITHUB_TOKEN=$GITHUB_TOKEN; \
-      echo "Setting GITHUB_TOKEN value as provided"; \
-    else \
-      export GITHUB_LIMIT=$(curl -s 'https://api.github.com/rate_limit' | jq '.rate .remaining'); \
-      echo "Current API rate limit https://api.github.com is ${GITHUB_LIMIT}"; \ 
-      if [ "${GITHUB_LIMIT}" -lt 10 ]; then \
-        printf "\033[0;31m\n\n\nRate limit on https://api.github.com is reached so in order to build this image, "; \
-        printf "the build argument GITHUB_TOKEN needs to be provided so build will not fail.\n\n\n\033[0m"; \
-        exit 1; \
-      else \
-        echo "GITHUB_TOKEN variable not set but https://api.github.com rate limit has enough slots"; \
-      fi \
-    fi && \
-    # install remaining packages
-    apk add --no-cache make gcc g++ python git bash supervisor && \
-    rm -rf /tmp/* /var/cache/apk/* && \
-    # Change version of Theia to specified in THEIA_VERSION
-    ${HOME}/versions.sh && rm ${HOME}/versions.sh && \
-    # Apply resolution section to the Theia package.json to use strict versions for Theia dependencies
-    node ${HOME}/resolutions-provider.js ${HOME}/package.json && \
-    # Generate node_modules, should be reused with default extensions
-    yarn && \
-    # Add default Theia extensions
-    node ${HOME}/add-extensions.js && \
-    # Launch `yarn` again to apply default extensions to the Theia node_modules
-    yarn && \
-    yarn theia clean && \
-    # Build Theia with all the extensions
-    yarn theia build && \
-    # Install Theia plugin generator
-    npm install -g yo @theia/generator-plugin && \
-    # Change permissions to allow editing of files for openshift user
-    find ${HOME} -exec sh -c "chgrp 0 {}; chmod g+rwX {}" \; && \
-    # Grant permissions for modifying supervisor log file
-    touch /var/log/supervisord.log && chmod g+rwX /var/log/supervisord.log && chgrp 0 /var/log/supervisord.log && \
-    yarn cache clean
+RUN ${HOME}/setup.sh
 
 ENTRYPOINT ["/usr/bin/supervisord", "-n", "-c", "/etc/supervisord.conf"]
+RUN apk update && apk add openjdk8

dockerfiles/theia/src/add-extensions.js

@@ -36,7 +36,7 @@ function addExtensions(extensions) {
     const extensionsToAdd = {};
     for (let extension of extensions) {
         const extensionName = extension["name"];
-        const extensionRootPath = EXTENSIONS_DIR + '/' + extensionName.replace(/\//g, '_');
+        let extensionRootPath = EXTENSIONS_DIR + '/' + extensionName.replace(/\//g, '_');
 
         const extensionType = extension["type"];
         switch(extensionType) {
@@ -49,8 +49,14 @@ function addExtensions(extensions) {
             default:
                 throw new Error(`Invalid extension type ${extensionType}.`);
         }
-        buildExtension(extensionRootPath);
-        extensionsToAdd[extensionName] = getBinaryPath(extensionRootPath, extensionName);
+        const buildFolder = extension["folder"];
+        const originalExtensionPath = extensionRootPath;
+        if (buildFolder) {
+            console.log('There is a build folder defined', buildFolder);
+            extensionRootPath = extensionRootPath + '/' + buildFolder;
+        }
+        buildExtension(extensionRootPath, originalExtensionPath);
+        extensionsToAdd[extensionName] = 'file://' + extensionRootPath;
     }
     console.log("Extension to add: ", extensionsToAdd);
     addExtensionsIntoDefaultPackageJson(extensionsToAdd);
@@ -98,38 +104,27 @@ function checkoutRepo(path, checkoutTarget) {
     }
 }
 
-function buildExtension(path) {
+function buildExtension(path, rootPath) {
     try {
+        console.log('Generate versions for extension: ', path);
+        spawnSync(`${DEFAULT_THEIA_ROOT}/versions.sh`, [], {cwd: `${path}`});
+        if (path !== rootPath) {
+            console.log("Removing parent files");
+            // cleanup files in root path
+            fs.unlinkSync(rootPath + '/package.json');
+            fs.unlinkSync(rootPath + '/lerna.json');
+        }
         console.log('Building extension: ', path);
         const nodeModulesPath = `${DEFAULT_THEIA_ROOT}/node_modules`;
         // build extension, but use Theia node_modules to reuse dependencies and prevent growing docker image.
-        spawnSync(`yarn`, ['--modules-folder', nodeModulesPath, '--global-folder', nodeModulesPath, '--cache-folder', nodeModulesPath], {cwd: `${path}`, stdio:[0,1,2]});
+        spawnSync(`yarn`, ['--modules-folder', nodeModulesPath, '--global-folder', nodeModulesPath], {cwd: `${path}`, stdio:[0,1,2]});
     } catch (error) {
+        console.error(error);
         console.error('Failed to build extension located in: ', path);
         process.exit(6);
     }
 }
 
-function getBinaryPath(extensionRoot, extensionName) {
-    const rootPackageJson = require(`${extensionRoot}/package.json`);
-    if ('theiaExtensions' in rootPackageJson) {
-        return extensionRoot;
-    }
-
-    const dirs = fs.readdirSync(extensionRoot).filter(item => !item.startsWith('.') && fs.lstatSync(extensionRoot + '/' + item).isDirectory());
-    for (let dirName of dirs) {
-        const extensionTargetDir = extensionRoot + '/' + dirName;
-        const packageJsonPath = extensionTargetDir + '/package.json';
-        if (fs.existsSync(packageJsonPath)) {
-            let packageJson = require(packageJsonPath);
-            if (packageJson['name'] === extensionName) {
-                return extensionRoot + '/' + dirName;
-            }
-        }
-    }
-    console.error('Failed to find folder with binaries for extension: ', extensionRoot);
-    process.exit(7);
-}
 
 function addExtensionsIntoDefaultPackageJson(extensions) {
     let theiaPackageJson = require(`${DEFAULT_THEIA_ROOT}/package.json`);

dockerfiles/theia/src/extensions.json

@@ -1,8 +1,30 @@
 {
     "extensions": [
+        {
+            "name": "github-extension",
+            "source": "https://github.com/eclipse/che-theia-github-plugin",
+            "folder": "github-extension",
+            "checkoutTo": "master",
+            "type": "git"
+        },
+        {
+            "name": "che-theia-ssh-extension",
+            "source": "https://github.com/eclipse/che-theia-ssh-plugin",
+            "folder": "ssh-extension",
+            "checkoutTo": "master",
+            "type": "git"
+        },
         {
             "name": "theia-machines-extension",
             "source": "https://github.com/eclipse/che-theia-machines-plugin.git",
+            "folder": "theia-machines-extension",
+            "checkoutTo": "master",
+            "type": "git"
+        },
+        {
+            "name": "@eclipse-che/theia-factory-extension",
+            "source": "https://github.com/eclipse/che-theia-factory-extension.git",
+            "folder": "che-theia-factory",
             "checkoutTo": "master",
             "type": "git"
         },
@@ -11,6 +33,13 @@
             "source": "https://github.com/eclipse/che-theia-hosted-plugin-manager-extension.git",
             "checkoutTo": "latest-deps",
             "type": "git"
+        },
+        {
+            "name": "@eclipse-che/theia-java-extension",
+            "source": "https://github.com/eclipse/che-theia-java-plugin",
+            "folder": "che-theia-java-extension",
+            "checkoutTo": "master",
+            "type": "git"
         }
     ]
 }

dockerfiles/theia/src/package.json

@@ -1,7 +1,9 @@
 {
     "private": true,
     "dependencies": {
+        "@theia/java": "latest",
         "@theia/typescript": "latest",
+        "@theia/editor": "latest",
         "@theia/navigator": "latest",
         "@theia/terminal": "latest",
         "@theia/outline-view": "latest",
@@ -9,9 +11,10 @@
         "@theia/git": "latest",
         "@theia/file-search": "latest",
         "@theia/markers": "latest",
+        "@theia/plugin-ext": "latest",
+        "@theia/plugin-ext-vscode": "latest",
         "@theia/extension-manager": "latest",
-        "@theia/messages": "latest",
-        "@eclipse-che/theia-factory-extension": "0.0.1-1530189818"
+        "@theia/messages": "latest"
     },
     "devDependencies": {
         "@theia/cli": "latest"

dockerfiles/theia/src/patches/0.3.12/001-regexp.patch

@@ -0,0 +1,27 @@
+From e358acd5eb55bab331fed8cb18ea935b4c9a2f8e Mon Sep 17 00:00:00 2001
+From: Florent BENOIT <fbenoit@redhat.com>
+Date: Tue, 3 Jul 2018 09:40:26 +0200
+Subject: [PATCH] Fix RegExp for starting hosted instance
+
+Change-Id: I010a83c47491da0f52c1d03c6e4df478a6e22a1c
+Signed-off-by: Florent BENOIT <fbenoit@redhat.com>
+---
+ packages/plugin-ext/src/hosted/node/hosted-plugin-manager.ts | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/packages/plugin-ext/src/hosted/node/hosted-plugin-manager.ts b/packages/plugin-ext/src/hosted/node/hosted-plugin-manager.ts
+index 3933626..9ea8fcb 100644
+--- a/packages/plugin-ext/src/hosted/node/hosted-plugin-manager.ts
++++ b/packages/plugin-ext/src/hosted/node/hosted-plugin-manager.ts
+@@ -64,7 +64,7 @@ export interface HostedPluginManager {
+ }
+ 
+ const HOSTED_INSTANCE_START_TIMEOUT_MS = 30000;
+-const THEIA_INSTANCE_REGEX = /.*Theia app listening on (.*)\. \[\].*/;
++const THEIA_INSTANCE_REGEX = /.*Theia app listening on (.*).*\./;
+ const PROCESS_OPTIONS = {
+     cwd: process.cwd(),
+     env: { ...process.env }
+-- 
+2.8.0
+

dockerfiles/theia/src/setup.sh

@@ -0,0 +1,140 @@
+#!/bin/sh
+
+# Install basic software used for checking github API rate limit
+apk update && apk add --no-cache curl jq ca-certificates openssl expect
+
+# update certificates
+update-ca-certificates
+
+# define in env variable GITHUB_TOKEN only if it is defined
+# else check if github rate limit is enough, else will abort requiring to set GITHUB_TOKEN value
+if [ "$GITHUB_TOKEN" != "" ]; then
+    export GITHUB_TOKEN=$GITHUB_TOKEN;
+    echo "Setting GITHUB_TOKEN value as provided";
+else
+    export GITHUB_LIMIT=$(curl -s 'https://api.github.com/rate_limit' | jq '.rate .remaining');
+    echo "Current API rate limit https://api.github.com is ${GITHUB_LIMIT}";
+    if [ "${GITHUB_LIMIT}" -lt 10 ]; then
+        printf "\033[0;31m\n\n\nRate limit on https://api.github.com is reached so in order to build this image, ";
+        printf "the build argument GITHUB_TOKEN needs to be provided so build will not fail.\n\n\n\033[0m";
+        exit 1;
+    else
+        echo "GITHUB_TOKEN variable not set but https://api.github.com rate limit has enough slots";
+    fi
+fi
+
+# install remaining packages
+# Include opendjk for Java support
+apk add --no-cache make gcc g++ python git bash supervisor openjdk8
+# Cleanup APK cache
+rm -rf /tmp/* /var/cache/apk/*
+
+# Install Yarn 1.7.0 (detect EPL 2.0 as a valid SPDX identifier)
+npm install -g yarn@1.7.0
+chmod u+x /usr/local/bin/yarn
+yarn --version
+
+
+# Clone specific tag of a Theia version
+git clone --branch v${THEIA_VERSION} https://github.com/theia-ide/theia ${HOME}/theia-source-code
+
+# Apply patches (if any)
+if [ -d "${HOME}/patches/${THEIA_VERSION}" ]; then
+    echo "Applying patches for Theia version ${THEIA_VERSION}";
+    for file in $(find "${HOME}/patches/${THEIA_VERSION}" -name '*.patch'); do
+        echo "Patching with ${file}";
+        cd ${HOME}/theia-source-code && patch -p1 < ${file};
+    done
+fi
+
+# Compile Theia
+cd ${HOME}/theia-source-code && yarn
+
+# add registry and start it
+npm install -g verdaccio
+mkdir ${HOME}/verdaccio
+cd ${HOME}/verdaccio
+verdaccio &
+sleep 3
+
+# Update registry URL to local one
+cd ${HOME}
+yarn config set registry http://localhost:4873
+npm config set registry http://localhost:4873
+
+
+# Create user for local registry
+export USERNAME=theia
+export PASSWORD=theia
+export EMAIL=che-theia@eclipse.org
+
+/usr/bin/expect <<EOD
+spawn npm adduser --registry http://localhost:4873
+expect {
+  "Username:" {send "$USERNAME\r"; exp_continue}
+  "Password:" {send "$PASSWORD\r"; exp_continue}
+  "Email: (this IS public)" {send "$EMAIL\r"; exp_continue}
+}
+EOD
+
+# Now go to source code of theia and publish it
+cd ${HOME}/theia-source-code
+
+# using 0.4 there to bump major version so we're sure to not download any 0.3.x dependencies
+# Set the version of Theia
+export THEIA_VERSION=0.4.1-che
+
+./node_modules/.bin/lerna publish --registry=http://localhost:4873 --exact --repo-version=${THEIA_VERSION} --skip-git --force-publish --npm-tag=latest  --yes
+cd ${HOME}
+
+# Code has been published, let's delete it
+rm -rf ${HOME}/theia-source-code
+
+# Change version of Theia to specified in THEIA_VERSION
+cd ${HOME} && ${HOME}/versions.sh 
+
+# Apply resolution section to the Theia package.json to use strict versions for Theia dependencies
+node ${HOME}/resolutions-provider.js ${HOME}/package.json
+
+# avoid issue with checksum of electron
+cd ${HOME} && npm install electron-packager -g
+
+# Add default Theia extensions
+cd ${HOME} && node ${HOME}/add-extensions.js
+
+# Build Theia with all the extensions
+cd ${HOME} && yarn && yarn theia build
+
+# Reset config registry
+npm config set registry https://registry.npmjs.org
+yarn config set registry https://registry.npmjs.org
+
+# install the latest theia generator of plug-in
+npm install -g yo @theia/generator-plugin
+mkdir -p ${HOME}/.config/insight-nodejs/
+chmod -R 777 ${HOME}/.config/
+
+# Change permissions to allow editing of files for openshift user
+find ${HOME} -exec sh -c "chgrp 0 {}; chmod g+rwX {}" \;
+# Grant permissions for modifying supervisor log file
+touch /var/log/supervisord.log && chmod g+rwX /var/log/supervisord.log && chgrp 0 /var/log/supervisord.log
+
+cd ${HOME}
+yarn cache clean
+
+# cleanup stuff installed temporary
+apk del curl jq expect
+npm uninstall -g verdaccio
+rm -rf ${HOME}/.config/verdaccio/
+
+# remove lock file as well
+rm ${HOME}/yarn.lock
+
+# remove installed scripts
+rm add-extensions.js resolutions-provider.js versions.sh setup.sh
+
+# remove extensions file
+rm extensions.json
+
+# use typescript globally (to have tsc/typescript working)
+npm install -g typescript@2.8.4

dockerfiles/theia/src/versions.sh

@@ -15,7 +15,7 @@ if [ -z "${THEIA_VERSION}" ]; then
 fi
 
 LATEST_VERSION="latest"
-PACKAGE_JSON_PATH="/home/theia/package.json"
+PACKAGE_JSON_PATH="./package.json"
 
 # modify  package json files according to provided THEIA_VERSION. Check if included packages are available before running sed
 PACKAGES_FULL=$(cat ${PACKAGE_JSON_PATH} | jq -r '.dependencies | keys | .[]')
@@ -25,7 +25,7 @@ for i in ${PACKAGES_FULL[@]}; do
       echo "${i} package with version ${THEIA_VERSION} not found. Using latest"
     else
       echo "Found ${i} package with version ${THEIA_VERSION}"
-      sed -i "s#\"${i}\": \"latest\"#\"${i}\": \"${THEIA_VERSION}\"#g" ${PACKAGE_JSON_PATH}
+      sed -i "s#\"${i}\": \".*\"#\"${i}\": \"${THEIA_VERSION}\"#g" ${PACKAGE_JSON_PATH}
     fi
 done