diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 6a9b050347..03f74a096b 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -46,3 +46,5 @@ wsagent/che-core-api-languageserver*/** @evidolob @dkuleshov wsagent/che-core-api-testing*/** @evidolob wsagent/che-wsagent-core/** @skabashnyuk wsagent/wsagent-local/** @skabashnyuk +wsagent/che-core-api-project/src/main/java/org/eclipse/che/api/search/server/** @skabashnyuk +wsagent/che-core-api-project/src/test/java/org/eclipse/che/api/search/** @skabashnyuk diff --git a/ide/che-core-ide-app/src/main/java/org/eclipse/che/ide/search/presentation/FoundOccurrenceNode.java b/ide/che-core-ide-app/src/main/java/org/eclipse/che/ide/search/presentation/FoundOccurrenceNode.java index b4edc5d398..7578505d3d 100644 --- a/ide/che-core-ide-app/src/main/java/org/eclipse/che/ide/search/presentation/FoundOccurrenceNode.java +++ b/ide/che-core-ide-app/src/main/java/org/eclipse/che/ide/search/presentation/FoundOccurrenceNode.java @@ -143,7 +143,7 @@ public class FoundOccurrenceNode extends AbstractTreeNode implements HasPresenta spanElement.setAttribute("debugFilePath", itemPath); SpanElement lineNumberElement = createSpanElement(); lineNumberElement.setInnerHTML( - String.valueOf(searchOccurrence.getLineNumber() + 1) + ":   "); + String.valueOf(searchOccurrence.getLineNumber()) + ":   "); spanElement.appendChild(lineNumberElement); SpanElement textElement = createSpanElement(); String phrase = searchOccurrence.getPhrase(); diff --git a/wsagent/che-core-api-project/pom.xml b/wsagent/che-core-api-project/pom.xml index 480cec50bd..05384d9c0d 100644 --- a/wsagent/che-core-api-project/pom.xml +++ b/wsagent/che-core-api-project/pom.xml @@ -101,7 +101,6 @@ org.eclipse.che.core che-core-api-project-shared - 6.1.0-SNAPSHOT org.eclipse.che.core @@ -123,10 +122,6 @@ org.eclipse.che.core che-core-commons-schedule - - org.eclipse.text - org.eclipse.text - org.slf4j slf4j-api @@ -172,11 +167,6 @@ che-core-commons-test test - - org.eclipse.equinox - common - test - org.everrest everrest-core diff --git a/wsagent/che-core-api-project/src/main/java/org/eclipse/che/api/project/server/ProjectService.java b/wsagent/che-core-api-project/src/main/java/org/eclipse/che/api/project/server/ProjectService.java index 0b763825ec..f434ef7cb5 100644 --- a/wsagent/che-core-api-project/src/main/java/org/eclipse/che/api/project/server/ProjectService.java +++ b/wsagent/che-core-api-project/src/main/java/org/eclipse/che/api/project/server/ProjectService.java @@ -666,7 +666,7 @@ public class ProjectService { @DefaultValue("-1") int maxItems, @ApiParam(value = "Skip count") @QueryParam("skipCount") int skipCount) - throws NotFoundException, ForbiddenException, ConflictException, ServerException { + throws NotFoundException, ServerException, BadRequestException { return getProjectServiceApi().search(wsPath, name, text, maxItems, skipCount); } diff --git a/wsagent/che-core-api-project/src/main/java/org/eclipse/che/api/project/server/impl/ProjectServiceApi.java b/wsagent/che-core-api-project/src/main/java/org/eclipse/che/api/project/server/impl/ProjectServiceApi.java index 05d8d3654b..4f15248947 100644 --- a/wsagent/che-core-api-project/src/main/java/org/eclipse/che/api/project/server/impl/ProjectServiceApi.java +++ b/wsagent/che-core-api-project/src/main/java/org/eclipse/che/api/project/server/impl/ProjectServiceApi.java @@ -75,20 +75,25 @@ import org.eclipse.che.api.project.shared.dto.SearchOccurrenceDto; import org.eclipse.che.api.project.shared.dto.SearchResultDto; import org.eclipse.che.api.project.shared.dto.SourceEstimation; import org.eclipse.che.api.project.shared.dto.TreeElement; +import org.eclipse.che.api.search.server.InvalidQueryException; +import org.eclipse.che.api.search.server.OffsetData; +import org.eclipse.che.api.search.server.QueryExecutionException; +import org.eclipse.che.api.search.server.QueryExpression; import org.eclipse.che.api.search.server.SearchResult; import org.eclipse.che.api.search.server.Searcher; -import org.eclipse.che.api.search.server.impl.LuceneSearcher; -import org.eclipse.che.api.search.server.impl.QueryExpression; import org.eclipse.che.api.search.server.impl.SearchResultEntry; import org.eclipse.che.api.workspace.shared.dto.ProjectConfigDto; import org.eclipse.che.api.workspace.shared.dto.SourceStorageDto; import org.eclipse.che.dto.server.DtoFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Project service REST API back end. This class' methods are called from the {@link * ProjectService}. */ public class ProjectServiceApi { + private static final Logger LOG = LoggerFactory.getLogger(ProjectServiceApi.class); private static Tika TIKA; @@ -602,9 +607,9 @@ public class ProjectServiceApi { */ public ProjectSearchResponseDto search( String wsPath, String name, String text, int maxItems, int skipCount) - throws NotFoundException, ForbiddenException, ConflictException, ServerException { + throws BadRequestException, ServerException, NotFoundException { if (skipCount < 0) { - throw new ConflictException(String.format("Invalid 'skipCount' parameter: %d.", skipCount)); + throw new BadRequestException(String.format("Invalid 'skipCount' parameter: %d.", skipCount)); } wsPath = absolutize(wsPath); @@ -617,11 +622,18 @@ public class ProjectServiceApi { .setSkipCount(skipCount) .setIncludePositions(true); - SearchResult result = searcher.search(expr); - List searchResultEntries = result.getResults(); - return DtoFactory.newDto(ProjectSearchResponseDto.class) - .withTotalHits(result.getTotalHits()) - .withItemReferences(prepareResults(searchResultEntries)); + try { + SearchResult result = searcher.search(expr); + List searchResultEntries = result.getResults(); + return DtoFactory.newDto(ProjectSearchResponseDto.class) + .withTotalHits(result.getTotalHits()) + .withItemReferences(prepareResults(searchResultEntries)); + } catch (InvalidQueryException e) { + throw new BadRequestException(e.getMessage()); + } catch (QueryExecutionException e) { + LOG.warn(e.getLocalizedMessage()); + throw new ServerException(e.getMessage()); + } } /** @@ -629,25 +641,25 @@ public class ProjectServiceApi { * found given text */ private List prepareResults(List searchResultEntries) - throws ServerException, NotFoundException { + throws NotFoundException { List results = new ArrayList<>(searchResultEntries.size()); for (SearchResultEntry searchResultEntry : searchResultEntries) { String path = searchResultEntry.getFilePath(); if (fsManager.existsAsFile(path)) { ItemReference asDto = fsDtoConverter.asDto(path); ItemReference itemReference = injectFileLinks(asDto); - List datas = searchResultEntry.getData(); + List datas = searchResultEntry.getData(); List searchOccurrences = new ArrayList<>(datas.size()); - for (LuceneSearcher.OffsetData data : datas) { + for (OffsetData data : datas) { SearchOccurrenceDto searchOccurrenceDto = DtoFactory.getInstance() .createDto(SearchOccurrenceDto.class) - .withPhrase(data.phrase) - .withScore(data.score) - .withStartOffset(data.startOffset) - .withEndOffset(data.endOffset) - .withLineNumber(data.lineNum) - .withLineContent(data.line); + .withPhrase(data.getPhrase()) + .withScore(data.getScore()) + .withStartOffset(data.getStartOffset()) + .withEndOffset(data.getEndOffset()) + .withLineNumber(data.getLineNum()) + .withLineContent(data.getLine()); searchOccurrences.add(searchOccurrenceDto); } SearchResultDto searchResultDto = DtoFactory.getInstance().createDto(SearchResultDto.class); @@ -680,7 +692,7 @@ public class ProjectServiceApi { try { return search(path, name, text, maxItems, skipCount); - } catch (ServerException | ConflictException | NotFoundException | ForbiddenException e) { + } catch (ServerException | NotFoundException | BadRequestException e) { throw new JsonRpcException(-27000, e.getMessage()); } } diff --git a/wsagent/che-core-api-project/src/main/java/org/eclipse/che/api/search/server/InvalidQueryException.java b/wsagent/che-core-api-project/src/main/java/org/eclipse/che/api/search/server/InvalidQueryException.java new file mode 100644 index 0000000000..d824e31c0e --- /dev/null +++ b/wsagent/che-core-api-project/src/main/java/org/eclipse/che/api/search/server/InvalidQueryException.java @@ -0,0 +1,19 @@ +/* + * Copyright (c) 2012-2018 Red Hat, Inc. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Red Hat, Inc. - initial API and implementation + */ +package org.eclipse.che.api.search.server; + +/** Razed in case if implementation specific format of search query is invalid */ +public class InvalidQueryException extends Exception { + + public InvalidQueryException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/wsagent/che-core-api-project/src/main/java/org/eclipse/che/api/search/server/OffsetData.java b/wsagent/che-core-api-project/src/main/java/org/eclipse/che/api/search/server/OffsetData.java new file mode 100644 index 0000000000..c63b1eed10 --- /dev/null +++ b/wsagent/che-core-api-project/src/main/java/org/eclipse/che/api/search/server/OffsetData.java @@ -0,0 +1,115 @@ +/* + * Copyright (c) 2012-2018 Red Hat, Inc. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Red Hat, Inc. - initial API and implementation + */ +package org.eclipse.che.api.search.server; + +public class OffsetData { + + private final String phrase; + private final int startOffset; + private final int endOffset; + private final float score; + private final int lineNum; + private final String line; + + public OffsetData( + String phrase, int startOffset, int endOffset, float score, int lineNum, String line) { + this.phrase = phrase; + this.startOffset = startOffset; + this.endOffset = endOffset; + this.score = score; + this.lineNum = lineNum; + this.line = line; + } + + public String getPhrase() { + return phrase; + } + + public int getStartOffset() { + return startOffset; + } + + public int getEndOffset() { + return endOffset; + } + + public float getScore() { + return score; + } + + public int getLineNum() { + return lineNum; + } + + public String getLine() { + return line; + } + + @Override + public String toString() { + return "OffsetData{" + + "phrase='" + + phrase + + '\'' + + ", startOffset=" + + startOffset + + ", endOffset=" + + endOffset + + ", score=" + + score + + ", lineNum=" + + lineNum + + ", line='" + + line + + '\'' + + '}'; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof OffsetData)) { + return false; + } + + OffsetData that = (OffsetData) o; + + if (getStartOffset() != that.getStartOffset()) { + return false; + } + if (getEndOffset() != that.getEndOffset()) { + return false; + } + if (Float.compare(that.getScore(), getScore()) != 0) { + return false; + } + if (getLineNum() != that.getLineNum()) { + return false; + } + if (getPhrase() != null ? !getPhrase().equals(that.getPhrase()) : that.getPhrase() != null) { + return false; + } + return getLine() != null ? getLine().equals(that.getLine()) : that.getLine() == null; + } + + @Override + public int hashCode() { + int result = getPhrase() != null ? getPhrase().hashCode() : 0; + result = 31 * result + getStartOffset(); + result = 31 * result + getEndOffset(); + result = 31 * result + (getScore() != +0.0f ? Float.floatToIntBits(getScore()) : 0); + result = 31 * result + getLineNum(); + result = 31 * result + (getLine() != null ? getLine().hashCode() : 0); + return result; + } +} diff --git a/wsagent/che-core-api-project/src/main/java/org/eclipse/che/api/search/server/QueryExecutionException.java b/wsagent/che-core-api-project/src/main/java/org/eclipse/che/api/search/server/QueryExecutionException.java new file mode 100644 index 0000000000..67f669090a --- /dev/null +++ b/wsagent/che-core-api-project/src/main/java/org/eclipse/che/api/search/server/QueryExecutionException.java @@ -0,0 +1,23 @@ +/* + * Copyright (c) 2012-2018 Red Hat, Inc. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Red Hat, Inc. - initial API and implementation + */ +package org.eclipse.che.api.search.server; + +/** Razed in something unrelated to user input happened during query execution */ +public class QueryExecutionException extends Exception { + + public QueryExecutionException(String message) { + super(message); + } + + public QueryExecutionException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/wsagent/che-core-api-project/src/main/java/org/eclipse/che/api/search/server/impl/QueryExpression.java b/wsagent/che-core-api-project/src/main/java/org/eclipse/che/api/search/server/QueryExpression.java similarity index 98% rename from wsagent/che-core-api-project/src/main/java/org/eclipse/che/api/search/server/impl/QueryExpression.java rename to wsagent/che-core-api-project/src/main/java/org/eclipse/che/api/search/server/QueryExpression.java index de851c4e9e..9cd3605399 100644 --- a/wsagent/che-core-api-project/src/main/java/org/eclipse/che/api/search/server/impl/QueryExpression.java +++ b/wsagent/che-core-api-project/src/main/java/org/eclipse/che/api/search/server/QueryExpression.java @@ -8,7 +8,7 @@ * Contributors: * Red Hat, Inc. - initial API and implementation */ -package org.eclipse.che.api.search.server.impl; +package org.eclipse.che.api.search.server; /** Container for parameters of query that executed by Searcher. */ public class QueryExpression { diff --git a/wsagent/che-core-api-project/src/main/java/org/eclipse/che/api/search/server/SearchResult.java b/wsagent/che-core-api-project/src/main/java/org/eclipse/che/api/search/server/SearchResult.java index 312e31ef93..7e630d3464 100644 --- a/wsagent/che-core-api-project/src/main/java/org/eclipse/che/api/search/server/SearchResult.java +++ b/wsagent/che-core-api-project/src/main/java/org/eclipse/che/api/search/server/SearchResult.java @@ -15,7 +15,6 @@ import static java.util.stream.Collectors.toList; import com.google.common.base.Optional; import java.util.List; -import org.eclipse.che.api.search.server.impl.QueryExpression; import org.eclipse.che.api.search.server.impl.SearchResultEntry; /** Result of executing {@link Searcher#search(QueryExpression)}. */ diff --git a/wsagent/che-core-api-project/src/main/java/org/eclipse/che/api/search/server/Searcher.java b/wsagent/che-core-api-project/src/main/java/org/eclipse/che/api/search/server/Searcher.java index 20029d563c..c725d523b6 100644 --- a/wsagent/che-core-api-project/src/main/java/org/eclipse/che/api/search/server/Searcher.java +++ b/wsagent/che-core-api-project/src/main/java/org/eclipse/che/api/search/server/Searcher.java @@ -11,9 +11,7 @@ package org.eclipse.che.api.search.server; import java.nio.file.Path; -import org.eclipse.che.api.core.NotFoundException; import org.eclipse.che.api.core.ServerException; -import org.eclipse.che.api.search.server.impl.QueryExpression; public interface Searcher { /** @@ -23,7 +21,7 @@ public interface Searcher { * @return results of search * @throws ServerException if an error occurs */ - SearchResult search(QueryExpression query) throws ServerException; + SearchResult search(QueryExpression query) throws InvalidQueryException, QueryExecutionException; /** * Add VirtualFile to index. @@ -31,7 +29,7 @@ public interface Searcher { * @param fsPath file to add * @throws ServerException if an error occurs */ - void add(Path fsPath) throws ServerException, NotFoundException; + void add(Path fsPath); /** * Delete VirtualFile from index. @@ -39,7 +37,7 @@ public interface Searcher { * @param fsPath path of VirtualFile * @throws ServerException if an error occurs */ - void delete(Path fsPath) throws ServerException, NotFoundException; + void delete(Path fsPath); /** * Updated indexed VirtualFile. @@ -47,5 +45,5 @@ public interface Searcher { * @param fsPath path of a file to update * @throws ServerException if an error occurs */ - void update(Path fsPath) throws ServerException, NotFoundException; + void update(Path fsPath); } diff --git a/wsagent/che-core-api-project/src/main/java/org/eclipse/che/api/search/server/consumers/IndexedFileCreateConsumer.java b/wsagent/che-core-api-project/src/main/java/org/eclipse/che/api/search/server/consumers/IndexedFileCreateConsumer.java index d5ffab99a2..1d2890b7d2 100644 --- a/wsagent/che-core-api-project/src/main/java/org/eclipse/che/api/search/server/consumers/IndexedFileCreateConsumer.java +++ b/wsagent/che-core-api-project/src/main/java/org/eclipse/che/api/search/server/consumers/IndexedFileCreateConsumer.java @@ -14,17 +14,11 @@ import java.nio.file.Path; import java.util.function.Consumer; import javax.inject.Inject; import javax.inject.Singleton; -import org.eclipse.che.api.core.NotFoundException; -import org.eclipse.che.api.core.ServerException; import org.eclipse.che.api.search.server.Searcher; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; @Singleton public class IndexedFileCreateConsumer implements Consumer { - private static final Logger LOG = LoggerFactory.getLogger(IndexedFileCreateConsumer.class); - private final Searcher searcher; @Inject @@ -34,10 +28,6 @@ public class IndexedFileCreateConsumer implements Consumer { @Override public void accept(Path fsPath) { - try { - searcher.add(fsPath); - } catch (ServerException | NotFoundException e) { - LOG.error("Issue happened during adding created file to index", e); - } + searcher.add(fsPath); } } diff --git a/wsagent/che-core-api-project/src/main/java/org/eclipse/che/api/search/server/consumers/IndexedFileDeleteConsumer.java b/wsagent/che-core-api-project/src/main/java/org/eclipse/che/api/search/server/consumers/IndexedFileDeleteConsumer.java index 7763729404..bbb448960c 100644 --- a/wsagent/che-core-api-project/src/main/java/org/eclipse/che/api/search/server/consumers/IndexedFileDeleteConsumer.java +++ b/wsagent/che-core-api-project/src/main/java/org/eclipse/che/api/search/server/consumers/IndexedFileDeleteConsumer.java @@ -14,17 +14,11 @@ import java.nio.file.Path; import java.util.function.Consumer; import javax.inject.Inject; import javax.inject.Singleton; -import org.eclipse.che.api.core.NotFoundException; -import org.eclipse.che.api.core.ServerException; import org.eclipse.che.api.search.server.Searcher; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; @Singleton public class IndexedFileDeleteConsumer implements Consumer { - private static final Logger LOG = LoggerFactory.getLogger(IndexedFileDeleteConsumer.class); - private final Searcher searcher; @Inject @@ -34,10 +28,6 @@ public class IndexedFileDeleteConsumer implements Consumer { @Override public void accept(Path fsPath) { - try { - searcher.delete(fsPath); - } catch (ServerException | NotFoundException e) { - LOG.error("Issue happened during removing deleted file from index", e); - } + searcher.delete(fsPath); } } diff --git a/wsagent/che-core-api-project/src/main/java/org/eclipse/che/api/search/server/consumers/IndexedFileUpdateConsumer.java b/wsagent/che-core-api-project/src/main/java/org/eclipse/che/api/search/server/consumers/IndexedFileUpdateConsumer.java index 8e74242987..eaba86e4f0 100644 --- a/wsagent/che-core-api-project/src/main/java/org/eclipse/che/api/search/server/consumers/IndexedFileUpdateConsumer.java +++ b/wsagent/che-core-api-project/src/main/java/org/eclipse/che/api/search/server/consumers/IndexedFileUpdateConsumer.java @@ -14,15 +14,10 @@ import java.nio.file.Path; import java.util.function.Consumer; import javax.inject.Inject; import javax.inject.Singleton; -import org.eclipse.che.api.core.NotFoundException; -import org.eclipse.che.api.core.ServerException; import org.eclipse.che.api.search.server.Searcher; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; @Singleton public class IndexedFileUpdateConsumer implements Consumer { - private static final Logger LOG = LoggerFactory.getLogger(IndexedFileDeleteConsumer.class); private final Searcher searcher; @@ -33,10 +28,6 @@ public class IndexedFileUpdateConsumer implements Consumer { @Override public void accept(Path fsPath) { - try { - searcher.update(fsPath); - } catch (ServerException | NotFoundException e) { - LOG.error("Issue happened during updating modified file in index", e); - } + searcher.update(fsPath); } } diff --git a/wsagent/che-core-api-project/src/main/java/org/eclipse/che/api/search/server/impl/LuceneSearcher.java b/wsagent/che-core-api-project/src/main/java/org/eclipse/che/api/search/server/impl/LuceneSearcher.java index 2fba9db14a..a3646e3519 100644 --- a/wsagent/che-core-api-project/src/main/java/org/eclipse/che/api/search/server/impl/LuceneSearcher.java +++ b/wsagent/che-core-api-project/src/main/java/org/eclipse/che/api/search/server/impl/LuceneSearcher.java @@ -11,13 +11,10 @@ package org.eclipse.che.api.search.server.impl; import static com.google.common.collect.Lists.newArrayList; -import static java.util.concurrent.Executors.newSingleThreadExecutor; -import static java.util.concurrent.TimeUnit.SECONDS; import static org.eclipse.che.api.fs.server.WsPathUtils.nameOf; -import static org.eclipse.che.commons.lang.IoUtil.deleteRecursive; +import com.google.common.annotations.VisibleForTesting; import com.google.common.io.CharStreams; -import com.google.common.util.concurrent.ThreadFactoryBuilder; import java.io.BufferedReader; import java.io.File; import java.io.FileInputStream; @@ -25,17 +22,19 @@ import java.io.IOException; import java.io.InputStreamReader; import java.io.Reader; import java.nio.charset.MalformedInputException; +import java.nio.file.FileVisitResult; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.PathMatcher; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; import java.util.ArrayList; import java.util.Collections; -import java.util.LinkedList; import java.util.List; +import java.util.Scanner; import java.util.Set; -import java.util.concurrent.ExecutorService; +import java.util.concurrent.CountDownLatch; import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; import javax.inject.Inject; import javax.inject.Named; import javax.inject.Singleton; @@ -48,6 +47,7 @@ import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; +import org.apache.lucene.document.SortedDocValuesField; import org.apache.lucene.document.StringField; import org.apache.lucene.document.TextField; import org.apache.lucene.index.IndexReader; @@ -57,6 +57,7 @@ import org.apache.lucene.index.Term; import org.apache.lucene.queryparser.classic.ParseException; import org.apache.lucene.queryparser.classic.QueryParser; import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.search.BooleanClause.Occur; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.PrefixQuery; @@ -64,23 +65,23 @@ import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.SearcherFactory; import org.apache.lucene.search.SearcherManager; +import org.apache.lucene.search.Sort; +import org.apache.lucene.search.SortField; +import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.highlight.QueryScorer; import org.apache.lucene.search.highlight.TokenSources; -import org.apache.lucene.store.Directory; import org.apache.lucene.store.FSDirectory; import org.apache.lucene.store.SingleInstanceLockFactory; -import org.apache.lucene.util.IOUtils; -import org.eclipse.che.api.core.NotFoundException; -import org.eclipse.che.api.core.ServerException; -import org.eclipse.che.api.core.util.FileCleaner; +import org.apache.lucene.util.BytesRef; import org.eclipse.che.api.fs.server.PathTransformer; +import org.eclipse.che.api.search.server.InvalidQueryException; +import org.eclipse.che.api.search.server.OffsetData; +import org.eclipse.che.api.search.server.QueryExecutionException; +import org.eclipse.che.api.search.server.QueryExpression; import org.eclipse.che.api.search.server.SearchResult; import org.eclipse.che.api.search.server.Searcher; -import org.eclipse.che.commons.lang.concurrent.LoggingUncaughtExceptionHandler; -import org.eclipse.jface.text.BadLocationException; -import org.eclipse.jface.text.IDocument; -import org.eclipse.jface.text.IRegion; +import org.eclipse.che.commons.schedule.ScheduleRate; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -88,6 +89,7 @@ import org.slf4j.LoggerFactory; * Lucene based searcher. * * @author andrew00x + * @author Sergii Kabashniuk */ @Singleton public class LuceneSearcher implements Searcher { @@ -100,107 +102,83 @@ public class LuceneSearcher implements Searcher { private static final String TEXT_FIELD = "text"; private final Set excludePatterns; - private final ExecutorService executor; private final File indexDirectory; private final PathTransformer pathTransformer; - private File root; - private IndexWriter luceneIndexWriter; - private SearcherManager searcherManager; - - private boolean closed = true; + private final File root; + private final IndexWriter luceneIndexWriter; + private final SearcherManager searcherManager; + private final Analyzer analyzer; + private final CountDownLatch initialIndexingLatch = new CountDownLatch(1); + private final Sort sort; @Inject - protected LuceneSearcher( + public LuceneSearcher( @Named("vfs.index_filter_matcher") Set excludePatterns, @Named("vfs.local.fs_index_root_dir") File indexDirectory, @Named("che.user.workspaces.storage") File root, - PathTransformer pathTransformer) { + PathTransformer pathTransformer) + throws IOException { + + if (indexDirectory.exists()) { + if (indexDirectory.isFile()) { + throw new IOException("Wrong configuration `vfs.local.fs_index_root_dir` is a file"); + } + } else { + Files.createDirectories(indexDirectory.toPath()); + } + this.indexDirectory = indexDirectory; this.root = root; this.excludePatterns = excludePatterns; this.pathTransformer = pathTransformer; - - executor = - newSingleThreadExecutor( - new ThreadFactoryBuilder() - .setDaemon(true) - .setUncaughtExceptionHandler(LoggingUncaughtExceptionHandler.getInstance()) - .setNameFormat("LuceneSearcherInitThread") - .build()); + this.analyzer = + CustomAnalyzer.builder() + .withTokenizer(WhitespaceTokenizerFactory.class) + .addTokenFilter(LowerCaseFilterFactory.class) + .build(); + this.luceneIndexWriter = + new IndexWriter( + FSDirectory.open(indexDirectory.toPath(), new SingleInstanceLockFactory()), + new IndexWriterConfig(analyzer)); + this.searcherManager = + new SearcherManager(luceneIndexWriter, true, true, new SearcherFactory()); + this.sort = new Sort(SortField.FIELD_SCORE, new SortField(PATH_FIELD, SortField.Type.STRING)); } @PostConstruct - private void initialize() throws ServerException { - doInitialize(); - if (!executor.isShutdown()) { - executor.execute( - () -> { - try { - addDirectory(root.toPath()); - } catch (ServerException e) { - LOG.error(e.getMessage()); - } - }); - } + @VisibleForTesting + void initialize() { + Thread initializer = + new Thread( + () -> { + try { + long start = System.currentTimeMillis(); + add(root.toPath()); + LOG.info( + "Initial indexing complete after {} msec ", System.currentTimeMillis() - start); + } finally { + initialIndexingLatch.countDown(); + } + }); + initializer.setName("LuceneSearcherInitThread"); + initializer.setDaemon(true); + initializer.start(); } - @PreDestroy - private void terminate() { - doTerminate(); - executor.shutdown(); - try { - if (!executor.awaitTermination(5, SECONDS)) { - executor.shutdownNow(); - } - } catch (InterruptedException ie) { - executor.shutdownNow(); - } + @VisibleForTesting + CountDownLatch getInitialIndexingLatch() { + return initialIndexingLatch; } - private Analyzer makeAnalyzer() throws IOException { - return CustomAnalyzer.builder() - .withTokenizer(WhitespaceTokenizerFactory.class) - .addTokenFilter(LowerCaseFilterFactory.class) - .build(); - } - - private Directory makeDirectory() throws ServerException { - try { - Files.createDirectories(indexDirectory.toPath()); - return FSDirectory.open(indexDirectory.toPath(), new SingleInstanceLockFactory()); - } catch (IOException e) { - throw new ServerException(e); - } - } - - private void doInitialize() throws ServerException { - try { - luceneIndexWriter = new IndexWriter(makeDirectory(), new IndexWriterConfig(makeAnalyzer())); - searcherManager = new SearcherManager(luceneIndexWriter, true, true, new SearcherFactory()); - closed = false; - } catch (IOException e) { - throw new ServerException(e); - } - } - - private void doTerminate() { - if (!closed) { - try { - IOUtils.close(luceneIndexWriter, luceneIndexWriter.getDirectory(), searcherManager); - if (!deleteRecursive(indexDirectory)) { - LOG.warn("Unable delete index directory '{}', add it in FileCleaner", indexDirectory); - FileCleaner.addFile(indexDirectory); - } - } catch (IOException e) { - LOG.error(e.getMessage(), e); - } - closed = true; - } + @ScheduleRate(period = 30, initialDelay = 30) + private void commitIndex() throws IOException { + luceneIndexWriter.commit(); } @Override - public SearchResult search(QueryExpression query) throws ServerException { + public SearchResult search(QueryExpression query) + throws InvalidQueryException, QueryExecutionException { IndexSearcher luceneSearcher = null; try { final long startTime = System.currentTimeMillis(); @@ -217,7 +195,7 @@ public class LuceneSearcher implements Searcher { final int numDocs = query.getMaxItems() > 0 ? Math.min(query.getMaxItems(), RESULT_LIMIT) : RESULT_LIMIT; - TopDocs topDocs = luceneSearcher.searchAfter(after, luceneQuery, numDocs); + TopDocs topDocs = luceneSearcher.searchAfter(after, luceneQuery, numDocs, sort, true, true); final long totalHitsNum = topDocs.totalHits; List results = newArrayList(); @@ -263,7 +241,7 @@ public class LuceneSearcher implements Searcher { endOffset = offsetAtt.endOffset(); if ((endOffset > txt.length()) || (startOffset > txt.length())) { - throw new ServerException( + throw new QueryExecutionException( "Token " + termAtt.toString() + " exceeds length of provided text size " @@ -272,25 +250,29 @@ public class LuceneSearcher implements Searcher { float res = queryScorer.getTokenScore(); if (res > 0.0F && startOffset <= endOffset) { - try { - IDocument document = new org.eclipse.jface.text.Document(txt); - int lineNum = document.getLineOfOffset(startOffset); - IRegion lineInfo = document.getLineInformation(lineNum); - String foundLine = document.get(lineInfo.getOffset(), lineInfo.getLength()); - String tokenText = document.get(startOffset, endOffset - startOffset); + String tokenText = txt.substring(startOffset, endOffset); + Scanner sc = new Scanner(txt); + int lineNum = 1; + long len = 0; + String foundLine = ""; + while (sc.hasNextLine()) { + foundLine = sc.nextLine(); - offsetData.add( - new OffsetData( - tokenText, startOffset, endOffset, docId, res, lineNum, foundLine)); - } catch (BadLocationException e) { - LOG.error(e.getLocalizedMessage(), e); - throw new ServerException("Can not provide data for token " + termAtt.toString()); + len += foundLine.length(); + if (len > startOffset) { + break; + } + lineNum++; } + offsetData.add( + new OffsetData(tokenText, startOffset, endOffset, res, lineNum, foundLine)); } } } } + String filePath = doc.getField(PATH_FIELD).stringValue(); + LOG.debug("Doc {} path {} score {} ", docId, filePath, scoreDoc.score); results.add(new SearchResultEntry(filePath, offsetData)); } @@ -309,8 +291,10 @@ public class LuceneSearcher implements Searcher { .withNextPageQueryExpression(nextPageQueryExpression) .withElapsedTimeMillis(elapsedTimeMillis) .build(); - } catch (IOException | ParseException e) { - throw new ServerException(e.getMessage(), e); + } catch (ParseException e) { + throw new InvalidQueryException(e.getMessage(), e); + } catch (IOException e) { + throw new QueryExecutionException(e.getMessage(), e); } finally { try { searcherManager.release(luceneSearcher); @@ -329,12 +313,12 @@ public class LuceneSearcher implements Searcher { luceneQueryBuilder.add(new PrefixQuery(new Term(PATH_FIELD, path)), BooleanClause.Occur.MUST); } if (name != null) { - QueryParser qParser = new QueryParser(NAME_FIELD, makeAnalyzer()); + QueryParser qParser = new QueryParser(NAME_FIELD, analyzer); qParser.setAllowLeadingWildcard(true); luceneQueryBuilder.add(qParser.parse(name), BooleanClause.Occur.MUST); } if (text != null) { - QueryParser qParser = new QueryParser(TEXT_FIELD, makeAnalyzer()); + QueryParser qParser = new QueryParser(TEXT_FIELD, analyzer); qParser.setAllowLeadingWildcard(true); luceneQueryBuilder.add(qParser.parse(text), BooleanClause.Occur.MUST); } @@ -348,7 +332,7 @@ public class LuceneSearcher implements Searcher { int retrievedDocs = 0; TopDocs topDocs; do { - topDocs = luceneSearcher.searchAfter(scoreDoc, luceneQuery, readFrameSize); + topDocs = luceneSearcher.searchAfter(scoreDoc, luceneQuery, readFrameSize, sort, true, true); if (topDocs.scoreDocs.length > 0) { scoreDoc = topDocs.scoreDocs[topDocs.scoreDocs.length - 1]; } @@ -373,90 +357,36 @@ public class LuceneSearcher implements Searcher { } @Override - public final void add(Path fsPath) throws ServerException, NotFoundException { - if (fsPath.toFile().isDirectory()) { - addDirectory(fsPath); - } else { - addFile(fsPath); - } - } + public final void add(Path fsPath) { - private void addDirectory(Path fsPath) throws ServerException { - long start = System.currentTimeMillis(); - LinkedList queue = new LinkedList<>(); - queue.add(fsPath.toFile()); - int indexedFiles = 0; - while (!queue.isEmpty()) { - File folder = queue.pop(); - if (folder.exists() && folder.isDirectory()) { - File[] files = folder.listFiles(); - if (files == null) { - continue; - } - for (File child : files) { - if (child.isDirectory()) { - queue.push(child); - } else { - addFile(child.toPath()); - indexedFiles++; - } - } - } - } - - long end = System.currentTimeMillis(); - LOG.debug("Indexed {} files from {}, time: {} ms", indexedFiles, fsPath, (end - start)); - } - - private void addFile(Path fsPath) throws ServerException { - if (!fsPath.toFile().exists()) { - return; - } - - if (!isNotExcluded(fsPath)) { - return; - } - - String wsPath = pathTransformer.transform(fsPath); - - try (Reader reader = - new BufferedReader(new InputStreamReader(new FileInputStream(fsPath.toFile()), "utf-8"))) { - luceneIndexWriter.updateDocument( - new Term(PATH_FIELD, wsPath), createDocument(wsPath, reader)); - } catch (OutOfMemoryError oome) { - doTerminate(); - throw oome; - } catch (IOException e) { - throw new ServerException(e.getMessage(), e); - } - } - - @Override - public final void delete(Path fsPath) throws ServerException { - String wsPath = pathTransformer.transform(fsPath); try { - if (fsPath.toFile().isFile()) { - Term term = new Term(PATH_FIELD, wsPath); - luceneIndexWriter.deleteDocuments(term); + if (fsPath.toFile().isDirectory()) { + try { + Files.walkFileTree( + fsPath, + new SimpleFileVisitor() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { + addFile(file); + return FileVisitResult.CONTINUE; + } + }); + } catch (IOException ignore) { + LOG.warn("Not able to index {} because {} ", fsPath.toString(), ignore.getMessage()); + } } else { - Term term = new Term(PATH_FIELD, wsPath + '/'); - luceneIndexWriter.deleteDocuments(new PrefixQuery(term)); + addFile(fsPath); } - } catch (OutOfMemoryError oome) { - doTerminate(); - throw oome; + printStatistic(); } catch (IOException e) { - throw new ServerException(e.getMessage(), e); + LOG.warn( + "Can't commit changes to index for: {} because {} ", + fsPath.toAbsolutePath().toString(), + e.getMessage()); } } - @Override - public final void update(Path fsPath) throws ServerException { - String wsPath = pathTransformer.transform(fsPath); - doUpdate(new Term(PATH_FIELD, wsPath), fsPath); - } - - private void doUpdate(Term deleteTerm, Path fsPath) throws ServerException { + private void addFile(Path fsPath) { if (!fsPath.toFile().exists()) { return; } @@ -464,34 +394,74 @@ public class LuceneSearcher implements Searcher { if (!isNotExcluded(fsPath)) { return; } - String wsPath = pathTransformer.transform(fsPath); + LOG.debug("Adding file {} ", wsPath); try (Reader reader = new BufferedReader(new InputStreamReader(new FileInputStream(fsPath.toFile()), "utf-8"))) { - luceneIndexWriter.updateDocument(deleteTerm, createDocument(wsPath, reader)); - } catch (OutOfMemoryError oome) { - doTerminate(); - throw oome; - } catch (IOException e) { - throw new ServerException(e.getMessage(), e); + String name = nameOf(wsPath); + Document doc = new Document(); + doc.add(new StringField(PATH_FIELD, wsPath, Field.Store.YES)); + doc.add(new SortedDocValuesField(PATH_FIELD, new BytesRef(wsPath))); + doc.add(new TextField(NAME_FIELD, name, Field.Store.YES)); + try { + doc.add(new TextField(TEXT_FIELD, CharStreams.toString(reader), Field.Store.YES)); + } catch (MalformedInputException e) { + LOG.warn("Can't index file: {}", wsPath); + } + luceneIndexWriter.updateDocument(new Term(PATH_FIELD, wsPath), doc); + + } catch (IOException oome) { + LOG.warn("Can't index file: {}", wsPath); } } - private Document createDocument(String wsPath, Reader reader) throws ServerException { - String name = nameOf(wsPath); - Document doc = new Document(); - doc.add(new StringField(PATH_FIELD, wsPath, Field.Store.YES)); - doc.add(new TextField(NAME_FIELD, name, Field.Store.YES)); + @Override + public final void delete(Path fsPath) { + + String wsPath = pathTransformer.transform(fsPath); try { - doc.add(new TextField(TEXT_FIELD, CharStreams.toString(reader), Field.Store.YES)); - } catch (MalformedInputException e) { - LOG.warn("Can't index file: {}", wsPath); + + // Since in most cases this is post action there is no way to find out is this a file + // or directory. Lets try to delete both + BooleanQuery.Builder deleteFileOrFolder = new BooleanQuery.Builder(); + deleteFileOrFolder.setMinimumNumberShouldMatch(1); + deleteFileOrFolder.add(new TermQuery(new Term(PATH_FIELD, wsPath)), Occur.SHOULD); + deleteFileOrFolder.add(new PrefixQuery(new Term(PATH_FIELD, wsPath + "/")), Occur.SHOULD); + luceneIndexWriter.deleteDocuments(deleteFileOrFolder.build()); + printStatistic(); } catch (IOException e) { - LOG.error("Can't index file: {}", wsPath); - throw new ServerException(e.getLocalizedMessage(), e); + LOG.warn("Can't delete index for file: {}", wsPath); } - return doc; + } + + private void printStatistic() throws IOException { + if (LOG.isDebugEnabled()) { + IndexSearcher luceneSearcher = null; + try { + searcherManager.maybeRefresh(); + luceneSearcher = searcherManager.acquire(); + IndexReader reader = luceneSearcher.getIndexReader(); + LOG.debug( + "IndexReader numDocs={} numDeletedDocs={} maxDoc={} hasDeletions={}. Writer numDocs={} numRamDocs={} hasPendingMerges={} hasUncommittedChanges={} hasDeletions={}", + reader.numDocs(), + reader.numDeletedDocs(), + reader.maxDoc(), + reader.hasDeletions(), + luceneIndexWriter.numDocs(), + luceneIndexWriter.numRamDocs(), + luceneIndexWriter.hasPendingMerges(), + luceneIndexWriter.hasUncommittedChanges(), + luceneIndexWriter.hasDeletions()); + } finally { + searcherManager.release(luceneSearcher); + } + } + } + + @Override + public final void update(Path fsPath) { + addFile(fsPath); } private boolean isNotExcluded(Path fsPath) { @@ -502,32 +472,4 @@ public class LuceneSearcher implements Searcher { } return true; } - - public static class OffsetData { - - public String phrase; - public int startOffset; - public int endOffset; - public int docId; - public float score; - public int lineNum; - public String line; - - public OffsetData( - String phrase, - int startOffset, - int endOffset, - int docId, - float score, - int lineNum, - String line) { - this.phrase = phrase; - this.startOffset = startOffset; - this.endOffset = endOffset; - this.docId = docId; - this.score = score; - this.lineNum = lineNum; - this.line = line; - } - } } diff --git a/wsagent/che-core-api-project/src/main/java/org/eclipse/che/api/search/server/impl/SearchResultEntry.java b/wsagent/che-core-api-project/src/main/java/org/eclipse/che/api/search/server/impl/SearchResultEntry.java index d42dc8557b..ebd33a4a53 100644 --- a/wsagent/che-core-api-project/src/main/java/org/eclipse/che/api/search/server/impl/SearchResultEntry.java +++ b/wsagent/che-core-api-project/src/main/java/org/eclipse/che/api/search/server/impl/SearchResultEntry.java @@ -11,7 +11,7 @@ package org.eclipse.che.api.search.server.impl; import java.util.List; -import org.eclipse.che.api.search.server.impl.LuceneSearcher.OffsetData; +import org.eclipse.che.api.search.server.OffsetData; /** Single item in {@code SearchResult}. */ public class SearchResultEntry { @@ -32,4 +32,35 @@ public class SearchResultEntry { public String getFilePath() { return filePath; } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof SearchResultEntry)) { + return false; + } + + SearchResultEntry that = (SearchResultEntry) o; + + if (getFilePath() != null + ? !getFilePath().equals(that.getFilePath()) + : that.getFilePath() != null) { + return false; + } + return getData() != null ? getData().equals(that.getData()) : that.getData() == null; + } + + @Override + public int hashCode() { + int result = getFilePath() != null ? getFilePath().hashCode() : 0; + result = 31 * result + (getData() != null ? getData().hashCode() : 0); + return result; + } + + @Override + public String toString() { + return "SearchResultEntry{" + "filePath='" + filePath + '\'' + ", data=" + data + '}'; + } } diff --git a/wsagent/che-core-api-project/src/test/java/org/eclipse/che/api/search/SearcherTest.java b/wsagent/che-core-api-project/src/test/java/org/eclipse/che/api/search/SearcherTest.java new file mode 100644 index 0000000000..bc27548545 --- /dev/null +++ b/wsagent/che-core-api-project/src/test/java/org/eclipse/che/api/search/SearcherTest.java @@ -0,0 +1,501 @@ +/* + * Copyright (c) 2012-2018 Red Hat, Inc. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Red Hat, Inc. - initial API and implementation + */ +package org.eclipse.che.api.search; +/* + * Copyright (c) 2012-2017 Red Hat, Inc. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Red Hat, Inc. - initial API and implementation + */ + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; + +import com.google.common.base.Optional; +import com.google.common.collect.ImmutableList; +import com.google.common.io.Files; +import java.io.File; +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.PathMatcher; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import org.eclipse.che.api.fs.server.impl.RootAwarePathTransformer; +import org.eclipse.che.api.search.server.InvalidQueryException; +import org.eclipse.che.api.search.server.OffsetData; +import org.eclipse.che.api.search.server.QueryExecutionException; +import org.eclipse.che.api.search.server.QueryExpression; +import org.eclipse.che.api.search.server.SearchResult; +import org.eclipse.che.api.search.server.Searcher; +import org.eclipse.che.api.search.server.impl.LuceneSearcher; +import org.eclipse.che.api.search.server.impl.SearchResultEntry; +import org.eclipse.che.commons.lang.IoUtil; +import org.eclipse.che.commons.lang.NameGenerator; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +@SuppressWarnings("Duplicates") +public class SearcherTest { + public static final String[] TEST_CONTENT = { + "Apollo set several major human spaceflight milestones", + "Maybe you should think twice", + "To be or not to be beeeee lambergeeene\n or may be not to be \n or insidebigword_to_continuebigword end of line", + "In early 1961, direct ascent was generally the mission mode in favor at NASA", + "Time to think" + }; + + File indexDirectory; + File workspaceStorage; + Set excludePatterns; + Searcher searcher; + RootAwarePathTransformer pathTransformer; + ContentBuilder contentBuilder; + + @BeforeMethod + public void setUp() throws Exception { + indexDirectory = Files.createTempDir(); + IoUtil.deleteRecursive(indexDirectory); + workspaceStorage = Files.createTempDir(); + excludePatterns = Collections.emptySet(); + pathTransformer = new RootAwarePathTransformer(workspaceStorage); + searcher = + new LuceneSearcher(excludePatterns, indexDirectory, workspaceStorage, pathTransformer); + contentBuilder = new ContentBuilder(workspaceStorage.toPath()); + } + + @AfterMethod + public void tearDown() throws Exception { + IoUtil.deleteRecursive(indexDirectory); + IoUtil.deleteRecursive(workspaceStorage); + } + + @Test + public void shouldBeAbleToFindSingleFile() + throws InvalidQueryException, QueryExecutionException, IOException { + + // given + contentBuilder.createFolder("aaa").createFile("aaa.txt", TEST_CONTENT[1]); + // when + searcher.add(contentBuilder.getLastUpdatedFile()); + // then + assertFind("should", "/aaa/aaa.txt"); + } + + @Test + public void shouldBeAbleToFindTwoFilesAddedAsSingleDirectory() + throws InvalidQueryException, QueryExecutionException, IOException { + // given + contentBuilder + .createFolder("folder") + .createFile("xxx.txt", TEST_CONTENT[2]) + .createFile("zzz.txt", TEST_CONTENT[1]); + // when + searcher.add(contentBuilder.getCurrentFolder()); + // then + assertFind("be", "/folder/xxx.txt"); + assertFind("should", "/folder/zzz.txt"); + } + + @Test + public void shouldBeAbleToUpdateSingleFile() + throws InvalidQueryException, QueryExecutionException, IOException { + // given + contentBuilder.createFolder("aaa").createFile("aaa.txt", TEST_CONTENT[2]); + searcher.add(contentBuilder.getLastUpdatedFile()); + assertEmptyResult("should"); + // when + contentBuilder.createFile("aaa.txt", TEST_CONTENT[1]); + searcher.add(contentBuilder.getLastUpdatedFile()); + // then + assertFind("should", "/aaa/aaa.txt"); + } + + @Test + public void shouldBeAbleToDeleteSingleFile() + throws InvalidQueryException, QueryExecutionException, IOException { + // given + contentBuilder + .createFolder("folder") + .createFile("xxx.txt", TEST_CONTENT[2]) + .createFile("zzz.txt", TEST_CONTENT[1]); + searcher.add(contentBuilder.getCurrentFolder()); + contentBuilder + .takeWorkspceRoot() + .createFolder("aaa") + .createFile("aaa.txt1", TEST_CONTENT[3]) + .createFile("aaa.txt", TEST_CONTENT[2]); + searcher.add(contentBuilder.getCurrentFolder()); + assertFind("be", "/aaa/aaa.txt", "/folder/xxx.txt"); + + // when + contentBuilder.deleteFileInCurrentFolder("aaa.txt"); + searcher.delete(contentBuilder.getLastUpdatedFile()); + // then + assertFind("be", "/folder/xxx.txt"); + } + + @Test + public void shouldBeAbleToFindNumberWithComaInText() + throws InvalidQueryException, QueryExecutionException, IOException { + // given + contentBuilder + .createFolder("aaa") + .createFile("aaa.txt1", TEST_CONTENT[3]) + .createFile("aaa.txt", TEST_CONTENT[2]); + searcher.add(contentBuilder.getCurrentFolder()); + // when + // then + assertFind("1961,", "/aaa/aaa.txt1"); + } + + @Test + public void shouldBeAbleToFindTwoWordsInText() + throws InvalidQueryException, QueryExecutionException, IOException { + // given + contentBuilder + .createFolder("aaa") + .createFile("aaa.txt1", TEST_CONTENT[3]) + .createFile("aaa.txt", TEST_CONTENT[2]); + searcher.add(contentBuilder.getCurrentFolder()); + // when + // then + assertFind("was generally", "/aaa/aaa.txt1"); + } + + @Test + public void shouldBeAbleToDeleteFolder() + throws InvalidQueryException, QueryExecutionException, IOException { + // given + contentBuilder + .createFolder("folder") + .createFile("xxx.txt", TEST_CONTENT[2]) + .createFile("zzz.txt", TEST_CONTENT[1]); + searcher.add(contentBuilder.getCurrentFolder()); + contentBuilder + .takeWorkspceRoot() + .createFolder("aaa") + .createFile("aaa.txt1", TEST_CONTENT[3]) + .createFile("aaa.txt", TEST_CONTENT[2]); + searcher.add(contentBuilder.getCurrentFolder()); + assertFind("be", "/aaa/aaa.txt", "/folder/xxx.txt"); + assertFind("generally", "/aaa/aaa.txt1"); + + // when + searcher.delete(contentBuilder.getCurrentFolder()); + contentBuilder.deleteCurrentFolder(); + // then + assertFind("be", "/folder/xxx.txt"); + assertEmptyResult("generally"); + } + + @Test + public void shouldBeAbleToSearchByWordFragment() + throws InvalidQueryException, QueryExecutionException, IOException { + // given + contentBuilder + .createFolder("folder") + .createFile("xxx.txt", TEST_CONTENT[0]) + .createFile("yyy.txt", TEST_CONTENT[1]) + .createFile("zzz.txt", TEST_CONTENT[2]); + searcher.add(contentBuilder.getCurrentFolder()); + // when + // then + assertFind("*stone*", "/folder/xxx.txt"); + } + + @Test + public void shouldBeAbleToSearchByTextTermAndFileName() + throws InvalidQueryException, QueryExecutionException, IOException { + // given + contentBuilder + .createFolder("folder") + .createFile("xxx.txt", TEST_CONTENT[2]) + .createFile("yyy.txt", TEST_CONTENT[1]) + .createFile("zzz.txt", TEST_CONTENT[2]); + searcher.add(contentBuilder.getCurrentFolder()); + // when + // then + assertFind(new QueryExpression().setText("be").setName("xxx.txt"), "/folder/xxx.txt"); + } + + @Test + public void shouldBeAbleToSearchByFullTextPatternAndFileName() + throws InvalidQueryException, QueryExecutionException, IOException { + // given + contentBuilder + .createFolder("folder") + .createFile("xxx.txt", TEST_CONTENT[2]) + .createFile("yyy.txt", TEST_CONTENT[1]) + .createFile("zzz.txt", TEST_CONTENT[2]); + searcher.add(contentBuilder.getCurrentFolder()); + // when + // then + assertFind(new QueryExpression().setText("*be*").setName("xxx.txt"), "/folder/xxx.txt"); + } + + @Test + public void shouldBeAbleToSearchWithPositions() + throws InvalidQueryException, QueryExecutionException, IOException { + // given + contentBuilder + .createFolder("folder") + .createFile("xxx.txt", TEST_CONTENT[2]) + .createFile("yyy.txt", TEST_CONTENT[1]) + .createFile("zzz.txt", TEST_CONTENT[4]); + searcher.add(contentBuilder.getCurrentFolder()); + // when + + // then + + String[] lines = TEST_CONTENT[2].split("\\r?\\n"); + assertFind( + new QueryExpression().setText("*to*").setIncludePositions(true), + new SearchResultEntry( + "/folder/xxx.txt", + ImmutableList.of( + new OffsetData("To", 0, 2, 1.0f, 1, lines[0]), + new OffsetData("to", 13, 15, 1.0f, 1, lines[0]), + new OffsetData("to", 54, 56, 1.0f, 2, lines[1]), + new OffsetData("insidebigword_to_continuebigword", 65, 97, 1.0f, 3, lines[2]))), + new SearchResultEntry( + "/folder/zzz.txt", + ImmutableList.of(new OffsetData("to", 5, 7, 1.0f, 1, TEST_CONTENT[4])))); + } + + @DataProvider + public Object[][] searchByName() { + return new Object[][] { + {"sameName.txt", "sameName.txt"}, + {"notCaseSensitive.txt", "notcasesensitive.txt"}, + {"fullName.txt", "full*"}, + {"file name.txt", "file name"}, + {"prefixFileName.txt", "prefixF*"}, + {"name.with.dot.txt", "name.With.Dot.txt"}, + }; + } + + @Test(dataProvider = "searchByName") + public void shouldSearchFileByName(String fileName, String searchedFileName) + throws InvalidQueryException, QueryExecutionException, IOException { + // given + contentBuilder + .createFolder("parent") + .createFolder("child") + .createFile(NameGenerator.generate(null, 10), TEST_CONTENT[3]) + .createFile(fileName, TEST_CONTENT[2]) + .createFile(NameGenerator.generate(null, 10), TEST_CONTENT[1]); + searcher.add(contentBuilder.getCurrentFolder()); + contentBuilder + .takeWorkspceRoot() + .createFolder("folder2") + .createFile(NameGenerator.generate(null, 10), TEST_CONTENT[2]) + .createFile(NameGenerator.generate(null, 10), TEST_CONTENT[2]); + searcher.add(contentBuilder.getCurrentFolder()); + // when + // then + assertFind(new QueryExpression().setName(searchedFileName), "/parent/child/" + fileName); + } + + @Test + public void shouldSearchByTextAndPath() + throws InvalidQueryException, QueryExecutionException, IOException { + // given + contentBuilder.createFolder("folder2").createFile("zzz.txt", TEST_CONTENT[2]); + searcher.add(contentBuilder.getCurrentFolder()); + contentBuilder + .takeWorkspceRoot() + .createFolder("folder1") + .createFolder("a") + .createFolder("B") + .createFile("xxx.txt", TEST_CONTENT[2]); + searcher.add(contentBuilder.getCurrentFolder()); + // when + // then + assertFind(new QueryExpression().setText("be").setPath("/folder1"), "/folder1/a/B/xxx.txt"); + } + + @Test + public void shouldSearchByTextAndPathAndFileName() + throws InvalidQueryException, QueryExecutionException, IOException { + contentBuilder + .createFolder("folder1") + .createFolder("a") + .createFolder("b") + .createFile("yyy.txt", TEST_CONTENT[2]) + .createFile("xxx.txt", TEST_CONTENT[2]); + searcher.add(contentBuilder.getCurrentFolder()); + + contentBuilder + .takeWorkspceRoot() + .createFolder("folder2") + .createFolder("a") + .createFolder("b") + .createFile("zzz.txt", TEST_CONTENT[2]); + searcher.add(contentBuilder.getCurrentFolder()); + // when + // then + assertFind( + new QueryExpression().setText("be").setPath("/folder1").setName("xxx.txt"), + "/folder1/a/b/xxx.txt"); + } + + @Test + public void shouldLimitsNumberOfSearchResultsWhenMaxItemIsSet() + throws InvalidQueryException, QueryExecutionException, IOException { + + // given + for (int i = 0; i < 125; i++) { + contentBuilder.createFile( + String.format("file%02d", i), TEST_CONTENT[i % TEST_CONTENT.length]); + } + searcher.add(contentBuilder.getCurrentFolder()); + + // when + SearchResult result = searcher.search(new QueryExpression().setText("mission").setMaxItems(5)); + // then + assertEquals(25, result.getTotalHits()); + assertEquals(5, result.getFilePaths().size()); + } + + @Test + public void shouldBeAbleToGeneratesQueryExpressionForRetrievingNextPageOfResults() + throws InvalidQueryException, QueryExecutionException, IOException { + // given + for (int i = 0; i < 125; i++) { + contentBuilder.createFile( + String.format("file%02d", i), TEST_CONTENT[i % TEST_CONTENT.length]); + } + searcher.add(contentBuilder.getCurrentFolder()); + + SearchResult result = + searcher.search(new QueryExpression().setText("spaceflight").setMaxItems(7)); + + assertEquals(result.getTotalHits(), 25); + + Optional optionalNextPageQueryExpression = result.getNextPageQueryExpression(); + assertTrue(optionalNextPageQueryExpression.isPresent()); + QueryExpression nextPageQueryExpression = optionalNextPageQueryExpression.get(); + assertEquals("spaceflight", nextPageQueryExpression.getText()); + assertEquals(7, nextPageQueryExpression.getSkipCount()); + assertEquals(7, nextPageQueryExpression.getMaxItems()); + } + + @Test + public void shouldBeAbleToRetrievesSearchResultWithPages() + throws InvalidQueryException, QueryExecutionException, IOException { + for (int i = 0; i < 125; i++) { + contentBuilder.createFile( + String.format("file%02d", i), TEST_CONTENT[i % TEST_CONTENT.length]); + } + searcher.add(contentBuilder.getCurrentFolder()); + + SearchResult firstPage = + searcher.search(new QueryExpression().setText("spaceflight").setMaxItems(8)); + assertEquals(firstPage.getFilePaths().size(), 8); + + QueryExpression nextPageQueryExpression = firstPage.getNextPageQueryExpression().get(); + nextPageQueryExpression.setMaxItems(100); + + SearchResult lastPage = searcher.search(nextPageQueryExpression); + assertEquals(lastPage.getFilePaths().size(), 17); + + assertTrue(Collections.disjoint(firstPage.getFilePaths(), lastPage.getFilePaths())); + } + + public void assertFind(QueryExpression query, SearchResultEntry... expectedResults) + throws InvalidQueryException, QueryExecutionException { + SearchResult result = searcher.search(query); + assertEquals(result.getResults(), Arrays.asList(expectedResults)); + } + + public void assertFind(QueryExpression query, String... expectedPaths) + throws InvalidQueryException, QueryExecutionException { + List paths = searcher.search(query).getFilePaths(); + assertEquals(paths, Arrays.asList(expectedPaths)); + } + + public void assertFind(String text, String... expectedPaths) + throws InvalidQueryException, QueryExecutionException { + assertFind(new QueryExpression().setText(text), expectedPaths); + } + + public void assertEmptyResult(QueryExpression query) + throws InvalidQueryException, QueryExecutionException { + List paths = searcher.search(query).getFilePaths(); + assertTrue(paths.isEmpty()); + } + + public void assertEmptyResult(String text) throws InvalidQueryException, QueryExecutionException { + assertEmptyResult(new QueryExpression().setText(text)); + } + + public static class ContentBuilder { + private final Path workspaceRoot; + private Path root; + private Path lastUpdatedFile; + + public ContentBuilder(Path root) { + this.workspaceRoot = root; + this.root = root; + } + + public ContentBuilder createFolder(String name) throws IOException { + this.root = Paths.get(this.root.toString(), name); + java.nio.file.Files.createDirectories(this.root); + return this; + } + + public ContentBuilder createFile(String name, String content) throws IOException { + this.lastUpdatedFile = Paths.get(this.root.toString(), name); + Files.write(content.getBytes(), lastUpdatedFile.toFile()); + return this; + } + + public Path getCurrentFolder() { + return this.root; + } + + public ContentBuilder takeParent() { + this.root = this.root.getParent(); + return this; + } + + public ContentBuilder takeWorkspceRoot() { + this.root = workspaceRoot; + return this; + } + + public Path getLastUpdatedFile() { + return lastUpdatedFile; + } + + public ContentBuilder deleteCurrentFolder() { + IoUtil.deleteRecursive(this.root.toFile()); + this.root = this.root.getParent(); + return this; + } + + public ContentBuilder deleteFileInCurrentFolder(String name) { + this.lastUpdatedFile = Paths.get(this.root.toString(), name); + this.lastUpdatedFile.toFile().delete(); + return this; + } + } +} diff --git a/wsagent/che-core-api-project/src/test/java/org/eclipse/che/api/search/server/impl/FSLuceneSearcherTest.java b/wsagent/che-core-api-project/src/test/java/org/eclipse/che/api/search/server/impl/FSLuceneSearcherTest.java new file mode 100644 index 0000000000..10874c25fb --- /dev/null +++ b/wsagent/che-core-api-project/src/test/java/org/eclipse/che/api/search/server/impl/FSLuceneSearcherTest.java @@ -0,0 +1,102 @@ +/* + * Copyright (c) 2012-2018 Red Hat, Inc. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Red Hat, Inc. - initial API and implementation + */ +package org.eclipse.che.api.search.server.impl; +/* + * Copyright (c) 2012-2017 Red Hat, Inc. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + * Red Hat, Inc. - initial API and implementation + */ + +import static com.google.common.collect.Lists.newArrayList; +import static org.eclipse.che.api.search.SearcherTest.TEST_CONTENT; +import static org.testng.Assert.assertEquals; + +import com.google.common.io.Files; +import java.io.File; +import java.nio.file.PathMatcher; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import org.eclipse.che.api.fs.server.impl.RootAwarePathTransformer; +import org.eclipse.che.api.search.SearcherTest.ContentBuilder; +import org.eclipse.che.api.search.server.QueryExpression; +import org.eclipse.che.commons.lang.IoUtil; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +@SuppressWarnings("Duplicates") +public class FSLuceneSearcherTest { + + File indexDirectory; + File workspaceStorage; + Set excludePatterns; + LuceneSearcher searcher; + RootAwarePathTransformer pathTransformer; + ContentBuilder contentBuilder; + + @BeforeMethod + public void setUp() throws Exception { + indexDirectory = Files.createTempDir(); + workspaceStorage = Files.createTempDir(); + excludePatterns = new HashSet<>(); + pathTransformer = new RootAwarePathTransformer(workspaceStorage); + searcher = + new LuceneSearcher(excludePatterns, indexDirectory, workspaceStorage, pathTransformer); + contentBuilder = new ContentBuilder(workspaceStorage.toPath()); + } + + @AfterMethod + public void tearDown() throws Exception { + IoUtil.deleteRecursive(indexDirectory); + IoUtil.deleteRecursive(workspaceStorage); + } + + @Test + public void shouldBeAbleToInitializesIndexForExistedFiles() throws Exception { + // given + contentBuilder + .createFolder("folder") + .createFile("xxx.txt", TEST_CONTENT[2]) + .createFile("zzz.txt", TEST_CONTENT[1]); + + // when + searcher.initialize(); + searcher.getInitialIndexingLatch().await(); + + // then + List paths = searcher.search(new QueryExpression().setText("think")).getFilePaths(); + assertEquals(newArrayList("/folder/zzz.txt"), paths); + } + + @Test + public void shouldBeAbleToExcludesFilesFromIndexWithFilter() throws Exception { + // given + contentBuilder + .createFolder("folder") + .createFile("xxx.txt", TEST_CONTENT[2]) + .createFile("yyy.txt", TEST_CONTENT[2]) + .createFile("zzz.txt", TEST_CONTENT[2]); + excludePatterns.add( + it -> it.toFile().isFile() && "yyy.txt".equals(it.getFileName().toString())); + searcher.add(contentBuilder.getCurrentFolder()); + + // when + List paths = searcher.search(new QueryExpression().setText("be")).getFilePaths(); + // then + assertEquals(newArrayList("/folder/xxx.txt", "/folder/zzz.txt"), paths); + } +}