seg_yinzy
/
che-server
mirror of https://github.com/eclipse-che/che-server.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LuceneSearcher redesign and fix memory leak (#8112)

6.19.x
Sergii Kabashniuk 2018-02-05 14:43:28 +02:00 committed by Mykhailo Kuznietsov
parent 4e6e8b30db
commit 70b87d7560
18 changed files with 1013 additions and 308 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/CODEOWNERS vendored
View File

@ -46,3 +46,5 @@ wsagent/che-core-api-languageserver*/** @evidolob @dkuleshov
wsagent/che-core-api-testing*/** @evidolob
wsagent/che-wsagent-core/** @skabashnyuk
wsagent/wsagent-local/** @skabashnyuk
wsagent/che-core-api-project/src/main/java/org/eclipse/che/api/search/server/** @skabashnyuk
wsagent/che-core-api-project/src/test/java/org/eclipse/che/api/search/** @skabashnyuk

2
ide/che-core-ide-app/src/main/java/org/eclipse/che/ide/search/presentation/FoundOccurrenceNode.java
View File

@ -143,7 +143,7 @@ public class FoundOccurrenceNode extends AbstractTreeNode implements HasPresenta
spanElement.setAttribute("debugFilePath", itemPath);
SpanElement lineNumberElement = createSpanElement();
lineNumberElement.setInnerHTML(
String.valueOf(searchOccurrence.getLineNumber() + 1) + ":   ");
String.valueOf(searchOccurrence.getLineNumber()) + ":   ");
spanElement.appendChild(lineNumberElement);
SpanElement textElement = createSpanElement();
String phrase = searchOccurrence.getPhrase();

10
wsagent/che-core-api-project/pom.xml
View File

@ -101,7 +101,6 @@
<dependency>
<groupId>org.eclipse.che.core</groupId>
<artifactId>che-core-api-project-shared</artifactId>
<version>6.1.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.eclipse.che.core</groupId>
@ -123,10 +122,6 @@
<groupId>org.eclipse.che.core</groupId>
<artifactId>che-core-commons-schedule</artifactId>
</dependency>
<dependency>
<groupId>org.eclipse.text</groupId>
<artifactId>org.eclipse.text</artifactId>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
@ -172,11 +167,6 @@
<artifactId>che-core-commons-test</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.eclipse.equinox</groupId>
<artifactId>common</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.everrest</groupId>
<artifactId>everrest-core</artifactId>

2
wsagent/che-core-api-project/src/main/java/org/eclipse/che/api/project/server/ProjectService.java
View File

@ -666,7 +666,7 @@ public class ProjectService {
@DefaultValue("-1")
int maxItems,
@ApiParam(value = "Skip count") @QueryParam("skipCount") int skipCount)
throws NotFoundException, ForbiddenException, ConflictException, ServerException {
throws NotFoundException, ServerException, BadRequestException {
return getProjectServiceApi().search(wsPath, name, text, maxItems, skipCount);
}

50
wsagent/che-core-api-project/src/main/java/org/eclipse/che/api/project/server/impl/ProjectServiceApi.java
View File

@ -75,20 +75,25 @@ import org.eclipse.che.api.project.shared.dto.SearchOccurrenceDto;
import org.eclipse.che.api.project.shared.dto.SearchResultDto;
import org.eclipse.che.api.project.shared.dto.SourceEstimation;
import org.eclipse.che.api.project.shared.dto.TreeElement;
import org.eclipse.che.api.search.server.InvalidQueryException;
import org.eclipse.che.api.search.server.OffsetData;
import org.eclipse.che.api.search.server.QueryExecutionException;
import org.eclipse.che.api.search.server.QueryExpression;
import org.eclipse.che.api.search.server.SearchResult;
import org.eclipse.che.api.search.server.Searcher;
import org.eclipse.che.api.search.server.impl.LuceneSearcher;
import org.eclipse.che.api.search.server.impl.QueryExpression;
import org.eclipse.che.api.search.server.impl.SearchResultEntry;
import org.eclipse.che.api.workspace.shared.dto.ProjectConfigDto;
import org.eclipse.che.api.workspace.shared.dto.SourceStorageDto;
import org.eclipse.che.dto.server.DtoFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Project service REST API back end. This class' methods are called from the {@link
* ProjectService}.
*/
public class ProjectServiceApi {
private static final Logger LOG = LoggerFactory.getLogger(ProjectServiceApi.class);
private static Tika TIKA;
@ -602,9 +607,9 @@ public class ProjectServiceApi {
*/
public ProjectSearchResponseDto search(
String wsPath, String name, String text, int maxItems, int skipCount)
throws NotFoundException, ForbiddenException, ConflictException, ServerException {
throws BadRequestException, ServerException, NotFoundException {
if (skipCount < 0) {
throw new ConflictException(String.format("Invalid 'skipCount' parameter: %d.", skipCount));
throw new BadRequestException(String.format("Invalid 'skipCount' parameter: %d.", skipCount));
}
wsPath = absolutize(wsPath);
@ -617,11 +622,18 @@ public class ProjectServiceApi {
.setSkipCount(skipCount)
.setIncludePositions(true);
SearchResult result = searcher.search(expr);
List<SearchResultEntry> searchResultEntries = result.getResults();
return DtoFactory.newDto(ProjectSearchResponseDto.class)
.withTotalHits(result.getTotalHits())
.withItemReferences(prepareResults(searchResultEntries));
try {
SearchResult result = searcher.search(expr);
List<SearchResultEntry> searchResultEntries = result.getResults();
return DtoFactory.newDto(ProjectSearchResponseDto.class)
.withTotalHits(result.getTotalHits())
.withItemReferences(prepareResults(searchResultEntries));
} catch (InvalidQueryException e) {
throw new BadRequestException(e.getMessage());
} catch (QueryExecutionException e) {
LOG.warn(e.getLocalizedMessage());
throw new ServerException(e.getMessage());
}
}
/**
@ -629,25 +641,25 @@ public class ProjectServiceApi {
* found given text
*/
private List<SearchResultDto> prepareResults(List<SearchResultEntry> searchResultEntries)
throws ServerException, NotFoundException {
throws NotFoundException {
List<SearchResultDto> results = new ArrayList<>(searchResultEntries.size());
for (SearchResultEntry searchResultEntry : searchResultEntries) {
String path = searchResultEntry.getFilePath();
if (fsManager.existsAsFile(path)) {
ItemReference asDto = fsDtoConverter.asDto(path);
ItemReference itemReference = injectFileLinks(asDto);
List<LuceneSearcher.OffsetData> datas = searchResultEntry.getData();
List<OffsetData> datas = searchResultEntry.getData();
List<SearchOccurrenceDto> searchOccurrences = new ArrayList<>(datas.size());
for (LuceneSearcher.OffsetData data : datas) {
for (OffsetData data : datas) {
SearchOccurrenceDto searchOccurrenceDto =
DtoFactory.getInstance()
.createDto(SearchOccurrenceDto.class)
.withPhrase(data.phrase)
.withScore(data.score)
.withStartOffset(data.startOffset)
.withEndOffset(data.endOffset)
.withLineNumber(data.lineNum)
.withLineContent(data.line);
.withPhrase(data.getPhrase())
.withScore(data.getScore())
.withStartOffset(data.getStartOffset())
.withEndOffset(data.getEndOffset())
.withLineNumber(data.getLineNum())
.withLineContent(data.getLine());
searchOccurrences.add(searchOccurrenceDto);
}
SearchResultDto searchResultDto = DtoFactory.getInstance().createDto(SearchResultDto.class);
@ -680,7 +692,7 @@ public class ProjectServiceApi {
try {
return search(path, name, text, maxItems, skipCount);
} catch (ServerException | ConflictException | NotFoundException | ForbiddenException e) {
} catch (ServerException | NotFoundException | BadRequestException e) {
throw new JsonRpcException(-27000, e.getMessage());
}
}

19
wsagent/che-core-api-project/src/main/java/org/eclipse/che/api/search/server/InvalidQueryException.java Normal file
View File

@ -0,0 +1,19 @@
/*
* Copyright (c) 2012-2018 Red Hat, Inc.
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at
* http://www.eclipse.org/legal/epl-v10.html
*
* Contributors:
* Red Hat, Inc. - initial API and implementation
*/
package org.eclipse.che.api.search.server;
/** Razed in case if implementation specific format of search query is invalid */
public class InvalidQueryException extends Exception {
public InvalidQueryException(String message, Throwable cause) {
super(message, cause);
}
}

115
wsagent/che-core-api-project/src/main/java/org/eclipse/che/api/search/server/OffsetData.java Normal file
View File

@ -0,0 +1,115 @@
/*
* Copyright (c) 2012-2018 Red Hat, Inc.
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at
* http://www.eclipse.org/legal/epl-v10.html
*
* Contributors:
* Red Hat, Inc. - initial API and implementation
*/
package org.eclipse.che.api.search.server;
public class OffsetData {
private final String phrase;
private final int startOffset;
private final int endOffset;
private final float score;
private final int lineNum;
private final String line;
public OffsetData(
String phrase, int startOffset, int endOffset, float score, int lineNum, String line) {
this.phrase = phrase;
this.startOffset = startOffset;
this.endOffset = endOffset;
this.score = score;
this.lineNum = lineNum;
this.line = line;
}
public String getPhrase() {
return phrase;
}
public int getStartOffset() {
return startOffset;
}
public int getEndOffset() {
return endOffset;
}
public float getScore() {
return score;
}
public int getLineNum() {
return lineNum;
}
public String getLine() {
return line;
}
@Override
public String toString() {
return "OffsetData{"
+ "phrase='"
+ phrase
+ '\''
+ ", startOffset="
+ startOffset
+ ", endOffset="
+ endOffset
+ ", score="
+ score
+ ", lineNum="
+ lineNum
+ ", line='"
+ line
+ '\''
+ '}';
}
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (!(o instanceof OffsetData)) {
return false;
}
OffsetData that = (OffsetData) o;
if (getStartOffset() != that.getStartOffset()) {
return false;
}
if (getEndOffset() != that.getEndOffset()) {
return false;
}
if (Float.compare(that.getScore(), getScore()) != 0) {
return false;
}
if (getLineNum() != that.getLineNum()) {
return false;
}
if (getPhrase() != null ? !getPhrase().equals(that.getPhrase()) : that.getPhrase() != null) {
return false;
}
return getLine() != null ? getLine().equals(that.getLine()) : that.getLine() == null;
}
@Override
public int hashCode() {
int result = getPhrase() != null ? getPhrase().hashCode() : 0;
result = 31 * result + getStartOffset();
result = 31 * result + getEndOffset();
result = 31 * result + (getScore() != +0.0f ? Float.floatToIntBits(getScore()) : 0);
result = 31 * result + getLineNum();
result = 31 * result + (getLine() != null ? getLine().hashCode() : 0);
return result;
}
}

23
wsagent/che-core-api-project/src/main/java/org/eclipse/che/api/search/server/QueryExecutionException.java Normal file
View File

@ -0,0 +1,23 @@
/*
* Copyright (c) 2012-2018 Red Hat, Inc.
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at
* http://www.eclipse.org/legal/epl-v10.html
*
* Contributors:
* Red Hat, Inc. - initial API and implementation
*/
package org.eclipse.che.api.search.server;
/** Razed in something unrelated to user input happened during query execution */
public class QueryExecutionException extends Exception {
public QueryExecutionException(String message) {
super(message);
}
public QueryExecutionException(String message, Throwable cause) {
super(message, cause);
}
}

2
wsagent/che-core-api-project/src/main/java/org/eclipse/che/api/search/server/impl/QueryExpression.java → wsagent/che-core-api-project/src/main/java/org/eclipse/che/api/search/server/QueryExpression.java
View File

@ -8,7 +8,7 @@
* Contributors:
* Red Hat, Inc. - initial API and implementation
*/
package org.eclipse.che.api.search.server.impl;
package org.eclipse.che.api.search.server;
/** Container for parameters of query that executed by Searcher. */
public class QueryExpression {

1
wsagent/che-core-api-project/src/main/java/org/eclipse/che/api/search/server/SearchResult.java
View File

@ -15,7 +15,6 @@ import static java.util.stream.Collectors.toList;
import com.google.common.base.Optional;
import java.util.List;
import org.eclipse.che.api.search.server.impl.QueryExpression;
import org.eclipse.che.api.search.server.impl.SearchResultEntry;
/** Result of executing {@link Searcher#search(QueryExpression)}. */

10
wsagent/che-core-api-project/src/main/java/org/eclipse/che/api/search/server/Searcher.java
View File

@ -11,9 +11,7 @@
package org.eclipse.che.api.search.server;
import java.nio.file.Path;
import org.eclipse.che.api.core.NotFoundException;
import org.eclipse.che.api.core.ServerException;
import org.eclipse.che.api.search.server.impl.QueryExpression;
public interface Searcher {
/**
@ -23,7 +21,7 @@ public interface Searcher {
* @return results of search
* @throws ServerException if an error occurs
*/
SearchResult search(QueryExpression query) throws ServerException;
SearchResult search(QueryExpression query) throws InvalidQueryException, QueryExecutionException;
/**
* Add VirtualFile to index.
@ -31,7 +29,7 @@ public interface Searcher {
* @param fsPath file to add
* @throws ServerException if an error occurs
*/
void add(Path fsPath) throws ServerException, NotFoundException;
void add(Path fsPath);
/**
* Delete VirtualFile from index.
@ -39,7 +37,7 @@ public interface Searcher {
* @param fsPath path of VirtualFile
* @throws ServerException if an error occurs
*/
void delete(Path fsPath) throws ServerException, NotFoundException;
void delete(Path fsPath);
/**
* Updated indexed VirtualFile.
@ -47,5 +45,5 @@ public interface Searcher {
* @param fsPath path of a file to update
* @throws ServerException if an error occurs
*/
void update(Path fsPath) throws ServerException, NotFoundException;
void update(Path fsPath);
}

12
wsagent/che-core-api-project/src/main/java/org/eclipse/che/api/search/server/consumers/IndexedFileCreateConsumer.java
View File

@ -14,17 +14,11 @@ import java.nio.file.Path;
import java.util.function.Consumer;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.eclipse.che.api.core.NotFoundException;
import org.eclipse.che.api.core.ServerException;
import org.eclipse.che.api.search.server.Searcher;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@Singleton
public class IndexedFileCreateConsumer implements Consumer<Path> {
private static final Logger LOG = LoggerFactory.getLogger(IndexedFileCreateConsumer.class);
private final Searcher searcher;
@Inject
@ -34,10 +28,6 @@ public class IndexedFileCreateConsumer implements Consumer<Path> {
@Override
public void accept(Path fsPath) {
try {
searcher.add(fsPath);
} catch (ServerException | NotFoundException e) {
LOG.error("Issue happened during adding created file to index", e);
}
searcher.add(fsPath);
}
}

12
wsagent/che-core-api-project/src/main/java/org/eclipse/che/api/search/server/consumers/IndexedFileDeleteConsumer.java
View File

@ -14,17 +14,11 @@ import java.nio.file.Path;
import java.util.function.Consumer;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.eclipse.che.api.core.NotFoundException;
import org.eclipse.che.api.core.ServerException;
import org.eclipse.che.api.search.server.Searcher;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@Singleton
public class IndexedFileDeleteConsumer implements Consumer<Path> {
private static final Logger LOG = LoggerFactory.getLogger(IndexedFileDeleteConsumer.class);
private final Searcher searcher;
@Inject
@ -34,10 +28,6 @@ public class IndexedFileDeleteConsumer implements Consumer<Path> {
@Override
public void accept(Path fsPath) {
try {
searcher.delete(fsPath);
} catch (ServerException | NotFoundException e) {
LOG.error("Issue happened during removing deleted file from index", e);
}
searcher.delete(fsPath);
}
}

11
wsagent/che-core-api-project/src/main/java/org/eclipse/che/api/search/server/consumers/IndexedFileUpdateConsumer.java
View File

@ -14,15 +14,10 @@ import java.nio.file.Path;
import java.util.function.Consumer;
import javax.inject.Inject;
import javax.inject.Singleton;
import org.eclipse.che.api.core.NotFoundException;
import org.eclipse.che.api.core.ServerException;
import org.eclipse.che.api.search.server.Searcher;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@Singleton
public class IndexedFileUpdateConsumer implements Consumer<Path> {
private static final Logger LOG = LoggerFactory.getLogger(IndexedFileDeleteConsumer.class);
private final Searcher searcher;
@ -33,10 +28,6 @@ public class IndexedFileUpdateConsumer implements Consumer<Path> {
@Override
public void accept(Path fsPath) {
try {
searcher.update(fsPath);
} catch (ServerException | NotFoundException e) {
LOG.error("Issue happened during updating modified file in index", e);
}
searcher.update(fsPath);
}
}

414
wsagent/che-core-api-project/src/main/java/org/eclipse/che/api/search/server/impl/LuceneSearcher.java
View File

@ -11,13 +11,10 @@
package org.eclipse.che.api.search.server.impl;
import static com.google.common.collect.Lists.newArrayList;
import static java.util.concurrent.Executors.newSingleThreadExecutor;
import static java.util.concurrent.TimeUnit.SECONDS;
import static org.eclipse.che.api.fs.server.WsPathUtils.nameOf;
import static org.eclipse.che.commons.lang.IoUtil.deleteRecursive;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.io.CharStreams;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
@ -25,17 +22,19 @@ import java.io.IOException;
import java.io.InputStreamReader;
import java.io.Reader;
import java.nio.charset.MalformedInputException;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.PathMatcher;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.ArrayList;
import java.util.Collections;
import java.util.LinkedList;
import java.util.List;
import java.util.Scanner;
import java.util.Set;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.CountDownLatch;
import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
import javax.inject.Inject;
import javax.inject.Named;
import javax.inject.Singleton;
@ -48,6 +47,7 @@ import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.SortedDocValuesField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.IndexReader;
@ -57,6 +57,7 @@ import org.apache.lucene.index.Term;
import org.apache.lucene.queryparser.classic.ParseException;
import org.apache.lucene.queryparser.classic.QueryParser;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.PrefixQuery;
@ -64,23 +65,23 @@ import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.SearcherFactory;
import org.apache.lucene.search.SearcherManager;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.search.highlight.QueryScorer;
import org.apache.lucene.search.highlight.TokenSources;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.store.SingleInstanceLockFactory;
import org.apache.lucene.util.IOUtils;
import org.eclipse.che.api.core.NotFoundException;
import org.eclipse.che.api.core.ServerException;
import org.eclipse.che.api.core.util.FileCleaner;
import org.apache.lucene.util.BytesRef;
import org.eclipse.che.api.fs.server.PathTransformer;
import org.eclipse.che.api.search.server.InvalidQueryException;
import org.eclipse.che.api.search.server.OffsetData;
import org.eclipse.che.api.search.server.QueryExecutionException;
import org.eclipse.che.api.search.server.QueryExpression;
import org.eclipse.che.api.search.server.SearchResult;
import org.eclipse.che.api.search.server.Searcher;
import org.eclipse.che.commons.lang.concurrent.LoggingUncaughtExceptionHandler;
import org.eclipse.jface.text.BadLocationException;
import org.eclipse.jface.text.IDocument;
import org.eclipse.jface.text.IRegion;
import org.eclipse.che.commons.schedule.ScheduleRate;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -88,6 +89,7 @@ import org.slf4j.LoggerFactory;
* Lucene based searcher.
*
* @author andrew00x
* @author Sergii Kabashniuk
*/
@Singleton
public class LuceneSearcher implements Searcher {
@ -100,107 +102,83 @@ public class LuceneSearcher implements Searcher {
private static final String TEXT_FIELD = "text";
private final Set<PathMatcher> excludePatterns;
private final ExecutorService executor;
private final File indexDirectory;
private final PathTransformer pathTransformer;
private File root;
private IndexWriter luceneIndexWriter;
private SearcherManager searcherManager;
private boolean closed = true;
private final File root;
private final IndexWriter luceneIndexWriter;
private final SearcherManager searcherManager;
private final Analyzer analyzer;
private final CountDownLatch initialIndexingLatch = new CountDownLatch(1);
private final Sort sort;
@Inject
protected LuceneSearcher(
public LuceneSearcher(
@Named("vfs.index_filter_matcher") Set<PathMatcher> excludePatterns,
@Named("vfs.local.fs_index_root_dir") File indexDirectory,
@Named("che.user.workspaces.storage") File root,
PathTransformer pathTransformer) {
PathTransformer pathTransformer)
throws IOException {
if (indexDirectory.exists()) {
if (indexDirectory.isFile()) {
throw new IOException("Wrong configuration `vfs.local.fs_index_root_dir` is a file");
}
} else {
Files.createDirectories(indexDirectory.toPath());
}
this.indexDirectory = indexDirectory;
this.root = root;
this.excludePatterns = excludePatterns;
this.pathTransformer = pathTransformer;
executor =
newSingleThreadExecutor(
new ThreadFactoryBuilder()
.setDaemon(true)
.setUncaughtExceptionHandler(LoggingUncaughtExceptionHandler.getInstance())
.setNameFormat("LuceneSearcherInitThread")
.build());
this.analyzer =
CustomAnalyzer.builder()
.withTokenizer(WhitespaceTokenizerFactory.class)
.addTokenFilter(LowerCaseFilterFactory.class)
.build();
this.luceneIndexWriter =
new IndexWriter(
FSDirectory.open(indexDirectory.toPath(), new SingleInstanceLockFactory()),
new IndexWriterConfig(analyzer));
this.searcherManager =
new SearcherManager(luceneIndexWriter, true, true, new SearcherFactory());
this.sort = new Sort(SortField.FIELD_SCORE, new SortField(PATH_FIELD, SortField.Type.STRING));
}
@PostConstruct
private void initialize() throws ServerException {
doInitialize();
if (!executor.isShutdown()) {
executor.execute(
() -> {
try {
addDirectory(root.toPath());
} catch (ServerException e) {
LOG.error(e.getMessage());
}
});
}
@VisibleForTesting
void initialize() {
Thread initializer =
new Thread(
() -> {
try {
long start = System.currentTimeMillis();
add(root.toPath());
LOG.info(
"Initial indexing complete after {} msec ", System.currentTimeMillis() - start);
} finally {
initialIndexingLatch.countDown();
}
});
initializer.setName("LuceneSearcherInitThread");
initializer.setDaemon(true);
initializer.start();
}
@PreDestroy
private void terminate() {
doTerminate();
executor.shutdown();
try {
if (!executor.awaitTermination(5, SECONDS)) {
executor.shutdownNow();
}
} catch (InterruptedException ie) {
executor.shutdownNow();
}
@VisibleForTesting
CountDownLatch getInitialIndexingLatch() {
return initialIndexingLatch;
}
private Analyzer makeAnalyzer() throws IOException {
return CustomAnalyzer.builder()
.withTokenizer(WhitespaceTokenizerFactory.class)
.addTokenFilter(LowerCaseFilterFactory.class)
.build();
}
private Directory makeDirectory() throws ServerException {
try {
Files.createDirectories(indexDirectory.toPath());
return FSDirectory.open(indexDirectory.toPath(), new SingleInstanceLockFactory());
} catch (IOException e) {
throw new ServerException(e);
}
}
private void doInitialize() throws ServerException {
try {
luceneIndexWriter = new IndexWriter(makeDirectory(), new IndexWriterConfig(makeAnalyzer()));
searcherManager = new SearcherManager(luceneIndexWriter, true, true, new SearcherFactory());
closed = false;
} catch (IOException e) {
throw new ServerException(e);
}
}
private void doTerminate() {
if (!closed) {
try {
IOUtils.close(luceneIndexWriter, luceneIndexWriter.getDirectory(), searcherManager);
if (!deleteRecursive(indexDirectory)) {
LOG.warn("Unable delete index directory '{}', add it in FileCleaner", indexDirectory);
FileCleaner.addFile(indexDirectory);
}
} catch (IOException e) {
LOG.error(e.getMessage(), e);
}
closed = true;
}
@ScheduleRate(period = 30, initialDelay = 30)
private void commitIndex() throws IOException {
luceneIndexWriter.commit();
}
@Override
public SearchResult search(QueryExpression query) throws ServerException {
public SearchResult search(QueryExpression query)
throws InvalidQueryException, QueryExecutionException {
IndexSearcher luceneSearcher = null;
try {
final long startTime = System.currentTimeMillis();
@ -217,7 +195,7 @@ public class LuceneSearcher implements Searcher {
final int numDocs =
query.getMaxItems() > 0 ? Math.min(query.getMaxItems(), RESULT_LIMIT) : RESULT_LIMIT;
TopDocs topDocs = luceneSearcher.searchAfter(after, luceneQuery, numDocs);
TopDocs topDocs = luceneSearcher.searchAfter(after, luceneQuery, numDocs, sort, true, true);
final long totalHitsNum = topDocs.totalHits;
List<SearchResultEntry> results = newArrayList();
@ -263,7 +241,7 @@ public class LuceneSearcher implements Searcher {
endOffset = offsetAtt.endOffset();
if ((endOffset > txt.length()) || (startOffset > txt.length())) {
throw new ServerException(
throw new QueryExecutionException(
"Token "
+ termAtt.toString()
+ " exceeds length of provided text size "
@ -272,25 +250,29 @@ public class LuceneSearcher implements Searcher {
float res = queryScorer.getTokenScore();
if (res > 0.0F && startOffset <= endOffset) {
try {
IDocument document = new org.eclipse.jface.text.Document(txt);
int lineNum = document.getLineOfOffset(startOffset);
IRegion lineInfo = document.getLineInformation(lineNum);
String foundLine = document.get(lineInfo.getOffset(), lineInfo.getLength());
String tokenText = document.get(startOffset, endOffset - startOffset);
String tokenText = txt.substring(startOffset, endOffset);
Scanner sc = new Scanner(txt);
int lineNum = 1;
long len = 0;
String foundLine = "";
while (sc.hasNextLine()) {
foundLine = sc.nextLine();
offsetData.add(
new OffsetData(
tokenText, startOffset, endOffset, docId, res, lineNum, foundLine));
} catch (BadLocationException e) {
LOG.error(e.getLocalizedMessage(), e);
throw new ServerException("Can not provide data for token " + termAtt.toString());
len += foundLine.length();
if (len > startOffset) {
break;
}
lineNum++;
}
offsetData.add(
new OffsetData(tokenText, startOffset, endOffset, res, lineNum, foundLine));
}
}
}
}
String filePath = doc.getField(PATH_FIELD).stringValue();
LOG.debug("Doc {} path {} score {} ", docId, filePath, scoreDoc.score);
results.add(new SearchResultEntry(filePath, offsetData));
}
@ -309,8 +291,10 @@ public class LuceneSearcher implements Searcher {
.withNextPageQueryExpression(nextPageQueryExpression)
.withElapsedTimeMillis(elapsedTimeMillis)
.build();
} catch (IOException | ParseException e) {
throw new ServerException(e.getMessage(), e);
} catch (ParseException e) {
throw new InvalidQueryException(e.getMessage(), e);
} catch (IOException e) {
throw new QueryExecutionException(e.getMessage(), e);
} finally {
try {
searcherManager.release(luceneSearcher);
@ -329,12 +313,12 @@ public class LuceneSearcher implements Searcher {
luceneQueryBuilder.add(new PrefixQuery(new Term(PATH_FIELD, path)), BooleanClause.Occur.MUST);
}
if (name != null) {
QueryParser qParser = new QueryParser(NAME_FIELD, makeAnalyzer());
QueryParser qParser = new QueryParser(NAME_FIELD, analyzer);
qParser.setAllowLeadingWildcard(true);
luceneQueryBuilder.add(qParser.parse(name), BooleanClause.Occur.MUST);
}
if (text != null) {
QueryParser qParser = new QueryParser(TEXT_FIELD, makeAnalyzer());
QueryParser qParser = new QueryParser(TEXT_FIELD, analyzer);
qParser.setAllowLeadingWildcard(true);
luceneQueryBuilder.add(qParser.parse(text), BooleanClause.Occur.MUST);
}
@ -348,7 +332,7 @@ public class LuceneSearcher implements Searcher {
int retrievedDocs = 0;
TopDocs topDocs;
do {
topDocs = luceneSearcher.searchAfter(scoreDoc, luceneQuery, readFrameSize);
topDocs = luceneSearcher.searchAfter(scoreDoc, luceneQuery, readFrameSize, sort, true, true);
if (topDocs.scoreDocs.length > 0) {
scoreDoc = topDocs.scoreDocs[topDocs.scoreDocs.length - 1];
}
@ -373,90 +357,36 @@ public class LuceneSearcher implements Searcher {
}
@Override
public final void add(Path fsPath) throws ServerException, NotFoundException {
if (fsPath.toFile().isDirectory()) {
addDirectory(fsPath);
} else {
addFile(fsPath);
}
}
public final void add(Path fsPath) {
private void addDirectory(Path fsPath) throws ServerException {
long start = System.currentTimeMillis();
LinkedList<File> queue = new LinkedList<>();
queue.add(fsPath.toFile());
int indexedFiles = 0;
while (!queue.isEmpty()) {
File folder = queue.pop();
if (folder.exists() && folder.isDirectory()) {
File[] files = folder.listFiles();
if (files == null) {
continue;
}
for (File child : files) {
if (child.isDirectory()) {
queue.push(child);
} else {
addFile(child.toPath());
indexedFiles++;
}
}
}
}
long end = System.currentTimeMillis();
LOG.debug("Indexed {} files from {}, time: {} ms", indexedFiles, fsPath, (end - start));
}
private void addFile(Path fsPath) throws ServerException {
if (!fsPath.toFile().exists()) {
return;
}
if (!isNotExcluded(fsPath)) {
return;
}
String wsPath = pathTransformer.transform(fsPath);
try (Reader reader =
new BufferedReader(new InputStreamReader(new FileInputStream(fsPath.toFile()), "utf-8"))) {
luceneIndexWriter.updateDocument(
new Term(PATH_FIELD, wsPath), createDocument(wsPath, reader));
} catch (OutOfMemoryError oome) {
doTerminate();
throw oome;
} catch (IOException e) {
throw new ServerException(e.getMessage(), e);
}
}
@Override
public final void delete(Path fsPath) throws ServerException {
String wsPath = pathTransformer.transform(fsPath);
try {
if (fsPath.toFile().isFile()) {
Term term = new Term(PATH_FIELD, wsPath);
luceneIndexWriter.deleteDocuments(term);
if (fsPath.toFile().isDirectory()) {
try {
Files.walkFileTree(
fsPath,
new SimpleFileVisitor<Path>() {
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) {
addFile(file);
return FileVisitResult.CONTINUE;
}
});
} catch (IOException ignore) {
LOG.warn("Not able to index {} because {} ", fsPath.toString(), ignore.getMessage());
}
} else {
Term term = new Term(PATH_FIELD, wsPath + '/');
luceneIndexWriter.deleteDocuments(new PrefixQuery(term));
addFile(fsPath);
}
} catch (OutOfMemoryError oome) {
doTerminate();
throw oome;
printStatistic();
} catch (IOException e) {
throw new ServerException(e.getMessage(), e);
LOG.warn(
"Can't commit changes to index for: {} because {} ",
fsPath.toAbsolutePath().toString(),
e.getMessage());
}
}
@Override
public final void update(Path fsPath) throws ServerException {
String wsPath = pathTransformer.transform(fsPath);
doUpdate(new Term(PATH_FIELD, wsPath), fsPath);
}
private void doUpdate(Term deleteTerm, Path fsPath) throws ServerException {
private void addFile(Path fsPath) {
if (!fsPath.toFile().exists()) {
return;
}
@ -464,34 +394,74 @@ public class LuceneSearcher implements Searcher {
if (!isNotExcluded(fsPath)) {
return;
}
String wsPath = pathTransformer.transform(fsPath);
LOG.debug("Adding file {} ", wsPath);
try (Reader reader =
new BufferedReader(new InputStreamReader(new FileInputStream(fsPath.toFile()), "utf-8"))) {
luceneIndexWriter.updateDocument(deleteTerm, createDocument(wsPath, reader));
} catch (OutOfMemoryError oome) {
doTerminate();
throw oome;
} catch (IOException e) {
throw new ServerException(e.getMessage(), e);
String name = nameOf(wsPath);
Document doc = new Document();
doc.add(new StringField(PATH_FIELD, wsPath, Field.Store.YES));
doc.add(new SortedDocValuesField(PATH_FIELD, new BytesRef(wsPath)));
doc.add(new TextField(NAME_FIELD, name, Field.Store.YES));
try {
doc.add(new TextField(TEXT_FIELD, CharStreams.toString(reader), Field.Store.YES));
} catch (MalformedInputException e) {
LOG.warn("Can't index file: {}", wsPath);
}
luceneIndexWriter.updateDocument(new Term(PATH_FIELD, wsPath), doc);
} catch (IOException oome) {
LOG.warn("Can't index file: {}", wsPath);
}
}
private Document createDocument(String wsPath, Reader reader) throws ServerException {
String name = nameOf(wsPath);
Document doc = new Document();
doc.add(new StringField(PATH_FIELD, wsPath, Field.Store.YES));
doc.add(new TextField(NAME_FIELD, name, Field.Store.YES));
@Override
public final void delete(Path fsPath) {
String wsPath = pathTransformer.transform(fsPath);
try {
doc.add(new TextField(TEXT_FIELD, CharStreams.toString(reader), Field.Store.YES));
} catch (MalformedInputException e) {
LOG.warn("Can't index file: {}", wsPath);
// Since in most cases this is post action there is no way to find out is this a file
// or directory. Lets try to delete both
BooleanQuery.Builder deleteFileOrFolder = new BooleanQuery.Builder();
deleteFileOrFolder.setMinimumNumberShouldMatch(1);
deleteFileOrFolder.add(new TermQuery(new Term(PATH_FIELD, wsPath)), Occur.SHOULD);
deleteFileOrFolder.add(new PrefixQuery(new Term(PATH_FIELD, wsPath + "/")), Occur.SHOULD);
luceneIndexWriter.deleteDocuments(deleteFileOrFolder.build());
printStatistic();
} catch (IOException e) {
LOG.error("Can't index file: {}", wsPath);
throw new ServerException(e.getLocalizedMessage(), e);
LOG.warn("Can't delete index for file: {}", wsPath);
}
return doc;
}
private void printStatistic() throws IOException {
if (LOG.isDebugEnabled()) {
IndexSearcher luceneSearcher = null;
try {
searcherManager.maybeRefresh();
luceneSearcher = searcherManager.acquire();
IndexReader reader = luceneSearcher.getIndexReader();
LOG.debug(
"IndexReader numDocs={} numDeletedDocs={} maxDoc={} hasDeletions={}. Writer numDocs={} numRamDocs={} hasPendingMerges={} hasUncommittedChanges={} hasDeletions={}",
reader.numDocs(),
reader.numDeletedDocs(),
reader.maxDoc(),
reader.hasDeletions(),
luceneIndexWriter.numDocs(),
luceneIndexWriter.numRamDocs(),
luceneIndexWriter.hasPendingMerges(),
luceneIndexWriter.hasUncommittedChanges(),
luceneIndexWriter.hasDeletions());
} finally {
searcherManager.release(luceneSearcher);
}
}
}
@Override
public final void update(Path fsPath) {
addFile(fsPath);
}
private boolean isNotExcluded(Path fsPath) {
@ -502,32 +472,4 @@ public class LuceneSearcher implements Searcher {
}
return true;
}
public static class OffsetData {
public String phrase;
public int startOffset;
public int endOffset;
public int docId;
public float score;
public int lineNum;
public String line;
public OffsetData(
String phrase,
int startOffset,
int endOffset,
int docId,
float score,
int lineNum,
String line) {
this.phrase = phrase;
this.startOffset = startOffset;
this.endOffset = endOffset;
this.docId = docId;
this.score = score;
this.lineNum = lineNum;
this.line = line;
}
}
}

33
wsagent/che-core-api-project/src/main/java/org/eclipse/che/api/search/server/impl/SearchResultEntry.java
View File

@ -11,7 +11,7 @@
package org.eclipse.che.api.search.server.impl;
import java.util.List;
import org.eclipse.che.api.search.server.impl.LuceneSearcher.OffsetData;
import org.eclipse.che.api.search.server.OffsetData;
/** Single item in {@code SearchResult}. */
public class SearchResultEntry {
@ -32,4 +32,35 @@ public class SearchResultEntry {
public String getFilePath() {
return filePath;
}
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (!(o instanceof SearchResultEntry)) {
return false;
}
SearchResultEntry that = (SearchResultEntry) o;
if (getFilePath() != null
? !getFilePath().equals(that.getFilePath())
: that.getFilePath() != null) {
return false;
}
return getData() != null ? getData().equals(that.getData()) : that.getData() == null;
}
@Override
public int hashCode() {
int result = getFilePath() != null ? getFilePath().hashCode() : 0;
result = 31 * result + (getData() != null ? getData().hashCode() : 0);
return result;
}
@Override
public String toString() {
return "SearchResultEntry{" + "filePath='" + filePath + '\'' + ", data=" + data + '}';
}
}

501
wsagent/che-core-api-project/src/test/java/org/eclipse/che/api/search/SearcherTest.java Normal file
View File

@ -0,0 +1,501 @@
/*
* Copyright (c) 2012-2018 Red Hat, Inc.
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at
* http://www.eclipse.org/legal/epl-v10.html
*
* Contributors:
* Red Hat, Inc. - initial API and implementation
*/
package org.eclipse.che.api.search;
/*
* Copyright (c) 2012-2017 Red Hat, Inc.
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at
* http://www.eclipse.org/legal/epl-v10.html
*
* Contributors:
* Red Hat, Inc. - initial API and implementation
*/
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertTrue;
import com.google.common.base.Optional;
import com.google.common.collect.ImmutableList;
import com.google.common.io.Files;
import java.io.File;
import java.io.IOException;
import java.nio.file.Path;
import java.nio.file.PathMatcher;
import java.nio.file.Paths;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Set;
import org.eclipse.che.api.fs.server.impl.RootAwarePathTransformer;
import org.eclipse.che.api.search.server.InvalidQueryException;
import org.eclipse.che.api.search.server.OffsetData;
import org.eclipse.che.api.search.server.QueryExecutionException;
import org.eclipse.che.api.search.server.QueryExpression;
import org.eclipse.che.api.search.server.SearchResult;
import org.eclipse.che.api.search.server.Searcher;
import org.eclipse.che.api.search.server.impl.LuceneSearcher;
import org.eclipse.che.api.search.server.impl.SearchResultEntry;
import org.eclipse.che.commons.lang.IoUtil;
import org.eclipse.che.commons.lang.NameGenerator;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
@SuppressWarnings("Duplicates")
public class SearcherTest {
public static final String[] TEST_CONTENT = {
"Apollo set several major human spaceflight milestones",
"Maybe you should think twice",
"To be or not to be beeeee lambergeeene\n or may be not to be \n or insidebigword_to_continuebigword end of line",
"In early 1961, direct ascent was generally the mission mode in favor at NASA",
"Time to think"
};
File indexDirectory;
File workspaceStorage;
Set<PathMatcher> excludePatterns;
Searcher searcher;
RootAwarePathTransformer pathTransformer;
ContentBuilder contentBuilder;
@BeforeMethod
public void setUp() throws Exception {
indexDirectory = Files.createTempDir();
IoUtil.deleteRecursive(indexDirectory);
workspaceStorage = Files.createTempDir();
excludePatterns = Collections.emptySet();
pathTransformer = new RootAwarePathTransformer(workspaceStorage);
searcher =
new LuceneSearcher(excludePatterns, indexDirectory, workspaceStorage, pathTransformer);
contentBuilder = new ContentBuilder(workspaceStorage.toPath());
}
@AfterMethod
public void tearDown() throws Exception {
IoUtil.deleteRecursive(indexDirectory);
IoUtil.deleteRecursive(workspaceStorage);
}
@Test
public void shouldBeAbleToFindSingleFile()
throws InvalidQueryException, QueryExecutionException, IOException {
// given
contentBuilder.createFolder("aaa").createFile("aaa.txt", TEST_CONTENT[1]);
// when
searcher.add(contentBuilder.getLastUpdatedFile());
// then
assertFind("should", "/aaa/aaa.txt");
}
@Test
public void shouldBeAbleToFindTwoFilesAddedAsSingleDirectory()
throws InvalidQueryException, QueryExecutionException, IOException {
// given
contentBuilder
.createFolder("folder")
.createFile("xxx.txt", TEST_CONTENT[2])
.createFile("zzz.txt", TEST_CONTENT[1]);
// when
searcher.add(contentBuilder.getCurrentFolder());
// then
assertFind("be", "/folder/xxx.txt");
assertFind("should", "/folder/zzz.txt");
}
@Test
public void shouldBeAbleToUpdateSingleFile()
throws InvalidQueryException, QueryExecutionException, IOException {
// given
contentBuilder.createFolder("aaa").createFile("aaa.txt", TEST_CONTENT[2]);
searcher.add(contentBuilder.getLastUpdatedFile());
assertEmptyResult("should");
// when
contentBuilder.createFile("aaa.txt", TEST_CONTENT[1]);
searcher.add(contentBuilder.getLastUpdatedFile());
// then
assertFind("should", "/aaa/aaa.txt");
}
@Test
public void shouldBeAbleToDeleteSingleFile()
throws InvalidQueryException, QueryExecutionException, IOException {
// given
contentBuilder
.createFolder("folder")
.createFile("xxx.txt", TEST_CONTENT[2])
.createFile("zzz.txt", TEST_CONTENT[1]);
searcher.add(contentBuilder.getCurrentFolder());
contentBuilder
.takeWorkspceRoot()
.createFolder("aaa")
.createFile("aaa.txt1", TEST_CONTENT[3])
.createFile("aaa.txt", TEST_CONTENT[2]);
searcher.add(contentBuilder.getCurrentFolder());
assertFind("be", "/aaa/aaa.txt", "/folder/xxx.txt");
// when
contentBuilder.deleteFileInCurrentFolder("aaa.txt");
searcher.delete(contentBuilder.getLastUpdatedFile());
// then
assertFind("be", "/folder/xxx.txt");
}
@Test
public void shouldBeAbleToFindNumberWithComaInText()
throws InvalidQueryException, QueryExecutionException, IOException {
// given
contentBuilder
.createFolder("aaa")
.createFile("aaa.txt1", TEST_CONTENT[3])
.createFile("aaa.txt", TEST_CONTENT[2]);
searcher.add(contentBuilder.getCurrentFolder());
// when
// then
assertFind("1961,", "/aaa/aaa.txt1");
}
@Test
public void shouldBeAbleToFindTwoWordsInText()
throws InvalidQueryException, QueryExecutionException, IOException {
// given
contentBuilder
.createFolder("aaa")
.createFile("aaa.txt1", TEST_CONTENT[3])
.createFile("aaa.txt", TEST_CONTENT[2]);
searcher.add(contentBuilder.getCurrentFolder());
// when
// then
assertFind("was generally", "/aaa/aaa.txt1");
}
@Test
public void shouldBeAbleToDeleteFolder()
throws InvalidQueryException, QueryExecutionException, IOException {
// given
contentBuilder
.createFolder("folder")
.createFile("xxx.txt", TEST_CONTENT[2])
.createFile("zzz.txt", TEST_CONTENT[1]);
searcher.add(contentBuilder.getCurrentFolder());
contentBuilder
.takeWorkspceRoot()
.createFolder("aaa")
.createFile("aaa.txt1", TEST_CONTENT[3])
.createFile("aaa.txt", TEST_CONTENT[2]);
searcher.add(contentBuilder.getCurrentFolder());
assertFind("be", "/aaa/aaa.txt", "/folder/xxx.txt");
assertFind("generally", "/aaa/aaa.txt1");
// when
searcher.delete(contentBuilder.getCurrentFolder());
contentBuilder.deleteCurrentFolder();
// then
assertFind("be", "/folder/xxx.txt");
assertEmptyResult("generally");
}
@Test
public void shouldBeAbleToSearchByWordFragment()
throws InvalidQueryException, QueryExecutionException, IOException {
// given
contentBuilder
.createFolder("folder")
.createFile("xxx.txt", TEST_CONTENT[0])
.createFile("yyy.txt", TEST_CONTENT[1])
.createFile("zzz.txt", TEST_CONTENT[2]);
searcher.add(contentBuilder.getCurrentFolder());
// when
// then
assertFind("*stone*", "/folder/xxx.txt");
}
@Test
public void shouldBeAbleToSearchByTextTermAndFileName()
throws InvalidQueryException, QueryExecutionException, IOException {
// given
contentBuilder
.createFolder("folder")
.createFile("xxx.txt", TEST_CONTENT[2])
.createFile("yyy.txt", TEST_CONTENT[1])
.createFile("zzz.txt", TEST_CONTENT[2]);
searcher.add(contentBuilder.getCurrentFolder());
// when
// then
assertFind(new QueryExpression().setText("be").setName("xxx.txt"), "/folder/xxx.txt");
}
@Test
public void shouldBeAbleToSearchByFullTextPatternAndFileName()
throws InvalidQueryException, QueryExecutionException, IOException {
// given
contentBuilder
.createFolder("folder")
.createFile("xxx.txt", TEST_CONTENT[2])
.createFile("yyy.txt", TEST_CONTENT[1])
.createFile("zzz.txt", TEST_CONTENT[2]);
searcher.add(contentBuilder.getCurrentFolder());
// when
// then
assertFind(new QueryExpression().setText("*be*").setName("xxx.txt"), "/folder/xxx.txt");
}
@Test
public void shouldBeAbleToSearchWithPositions()
throws InvalidQueryException, QueryExecutionException, IOException {
// given
contentBuilder
.createFolder("folder")
.createFile("xxx.txt", TEST_CONTENT[2])
.createFile("yyy.txt", TEST_CONTENT[1])
.createFile("zzz.txt", TEST_CONTENT[4]);
searcher.add(contentBuilder.getCurrentFolder());
// when
// then
String[] lines = TEST_CONTENT[2].split("\\r?\\n");
assertFind(
new QueryExpression().setText("*to*").setIncludePositions(true),
new SearchResultEntry(
"/folder/xxx.txt",
ImmutableList.of(
new OffsetData("To", 0, 2, 1.0f, 1, lines[0]),
new OffsetData("to", 13, 15, 1.0f, 1, lines[0]),
new OffsetData("to", 54, 56, 1.0f, 2, lines[1]),
new OffsetData("insidebigword_to_continuebigword", 65, 97, 1.0f, 3, lines[2]))),
new SearchResultEntry(
"/folder/zzz.txt",
ImmutableList.of(new OffsetData("to", 5, 7, 1.0f, 1, TEST_CONTENT[4]))));
}
@DataProvider
public Object[][] searchByName() {
return new Object[][] {
{"sameName.txt", "sameName.txt"},
{"notCaseSensitive.txt", "notcasesensitive.txt"},
{"fullName.txt", "full*"},
{"file name.txt", "file name"},
{"prefixFileName.txt", "prefixF*"},
{"name.with.dot.txt", "name.With.Dot.txt"},
};
}
@Test(dataProvider = "searchByName")
public void shouldSearchFileByName(String fileName, String searchedFileName)
throws InvalidQueryException, QueryExecutionException, IOException {
// given
contentBuilder
.createFolder("parent")
.createFolder("child")
.createFile(NameGenerator.generate(null, 10), TEST_CONTENT[3])
.createFile(fileName, TEST_CONTENT[2])
.createFile(NameGenerator.generate(null, 10), TEST_CONTENT[1]);
searcher.add(contentBuilder.getCurrentFolder());
contentBuilder
.takeWorkspceRoot()
.createFolder("folder2")
.createFile(NameGenerator.generate(null, 10), TEST_CONTENT[2])
.createFile(NameGenerator.generate(null, 10), TEST_CONTENT[2]);
searcher.add(contentBuilder.getCurrentFolder());
// when
// then
assertFind(new QueryExpression().setName(searchedFileName), "/parent/child/" + fileName);
}
@Test
public void shouldSearchByTextAndPath()
throws InvalidQueryException, QueryExecutionException, IOException {
// given
contentBuilder.createFolder("folder2").createFile("zzz.txt", TEST_CONTENT[2]);
searcher.add(contentBuilder.getCurrentFolder());
contentBuilder
.takeWorkspceRoot()
.createFolder("folder1")
.createFolder("a")
.createFolder("B")
.createFile("xxx.txt", TEST_CONTENT[2]);
searcher.add(contentBuilder.getCurrentFolder());
// when
// then
assertFind(new QueryExpression().setText("be").setPath("/folder1"), "/folder1/a/B/xxx.txt");
}
@Test
public void shouldSearchByTextAndPathAndFileName()
throws InvalidQueryException, QueryExecutionException, IOException {
contentBuilder
.createFolder("folder1")
.createFolder("a")
.createFolder("b")
.createFile("yyy.txt", TEST_CONTENT[2])
.createFile("xxx.txt", TEST_CONTENT[2]);
searcher.add(contentBuilder.getCurrentFolder());
contentBuilder
.takeWorkspceRoot()
.createFolder("folder2")
.createFolder("a")
.createFolder("b")
.createFile("zzz.txt", TEST_CONTENT[2]);
searcher.add(contentBuilder.getCurrentFolder());
// when
// then
assertFind(
new QueryExpression().setText("be").setPath("/folder1").setName("xxx.txt"),
"/folder1/a/b/xxx.txt");
}
@Test
public void shouldLimitsNumberOfSearchResultsWhenMaxItemIsSet()
throws InvalidQueryException, QueryExecutionException, IOException {
// given
for (int i = 0; i < 125; i++) {
contentBuilder.createFile(
String.format("file%02d", i), TEST_CONTENT[i % TEST_CONTENT.length]);
}
searcher.add(contentBuilder.getCurrentFolder());
// when
SearchResult result = searcher.search(new QueryExpression().setText("mission").setMaxItems(5));
// then
assertEquals(25, result.getTotalHits());
assertEquals(5, result.getFilePaths().size());
}
@Test
public void shouldBeAbleToGeneratesQueryExpressionForRetrievingNextPageOfResults()
throws InvalidQueryException, QueryExecutionException, IOException {
// given
for (int i = 0; i < 125; i++) {
contentBuilder.createFile(
String.format("file%02d", i), TEST_CONTENT[i % TEST_CONTENT.length]);
}
searcher.add(contentBuilder.getCurrentFolder());
SearchResult result =
searcher.search(new QueryExpression().setText("spaceflight").setMaxItems(7));
assertEquals(result.getTotalHits(), 25);
Optional<QueryExpression> optionalNextPageQueryExpression = result.getNextPageQueryExpression();
assertTrue(optionalNextPageQueryExpression.isPresent());
QueryExpression nextPageQueryExpression = optionalNextPageQueryExpression.get();
assertEquals("spaceflight", nextPageQueryExpression.getText());
assertEquals(7, nextPageQueryExpression.getSkipCount());
assertEquals(7, nextPageQueryExpression.getMaxItems());
}
@Test
public void shouldBeAbleToRetrievesSearchResultWithPages()
throws InvalidQueryException, QueryExecutionException, IOException {
for (int i = 0; i < 125; i++) {
contentBuilder.createFile(
String.format("file%02d", i), TEST_CONTENT[i % TEST_CONTENT.length]);
}
searcher.add(contentBuilder.getCurrentFolder());
SearchResult firstPage =
searcher.search(new QueryExpression().setText("spaceflight").setMaxItems(8));
assertEquals(firstPage.getFilePaths().size(), 8);
QueryExpression nextPageQueryExpression = firstPage.getNextPageQueryExpression().get();
nextPageQueryExpression.setMaxItems(100);
SearchResult lastPage = searcher.search(nextPageQueryExpression);
assertEquals(lastPage.getFilePaths().size(), 17);
assertTrue(Collections.disjoint(firstPage.getFilePaths(), lastPage.getFilePaths()));
}
public void assertFind(QueryExpression query, SearchResultEntry... expectedResults)
throws InvalidQueryException, QueryExecutionException {
SearchResult result = searcher.search(query);
assertEquals(result.getResults(), Arrays.asList(expectedResults));
}
public void assertFind(QueryExpression query, String... expectedPaths)
throws InvalidQueryException, QueryExecutionException {
List<String> paths = searcher.search(query).getFilePaths();
assertEquals(paths, Arrays.asList(expectedPaths));
}
public void assertFind(String text, String... expectedPaths)
throws InvalidQueryException, QueryExecutionException {
assertFind(new QueryExpression().setText(text), expectedPaths);
}
public void assertEmptyResult(QueryExpression query)
throws InvalidQueryException, QueryExecutionException {
List<String> paths = searcher.search(query).getFilePaths();
assertTrue(paths.isEmpty());
}
public void assertEmptyResult(String text) throws InvalidQueryException, QueryExecutionException {
assertEmptyResult(new QueryExpression().setText(text));
}
public static class ContentBuilder {
private final Path workspaceRoot;
private Path root;
private Path lastUpdatedFile;
public ContentBuilder(Path root) {
this.workspaceRoot = root;
this.root = root;
}
public ContentBuilder createFolder(String name) throws IOException {
this.root = Paths.get(this.root.toString(), name);
java.nio.file.Files.createDirectories(this.root);
return this;
}
public ContentBuilder createFile(String name, String content) throws IOException {
this.lastUpdatedFile = Paths.get(this.root.toString(), name);
Files.write(content.getBytes(), lastUpdatedFile.toFile());
return this;
}
public Path getCurrentFolder() {
return this.root;
}
public ContentBuilder takeParent() {
this.root = this.root.getParent();
return this;
}
public ContentBuilder takeWorkspceRoot() {
this.root = workspaceRoot;
return this;
}
public Path getLastUpdatedFile() {
return lastUpdatedFile;
}
public ContentBuilder deleteCurrentFolder() {
IoUtil.deleteRecursive(this.root.toFile());
this.root = this.root.getParent();
return this;
}
public ContentBuilder deleteFileInCurrentFolder(String name) {
this.lastUpdatedFile = Paths.get(this.root.toString(), name);
this.lastUpdatedFile.toFile().delete();
return this;
}
}
}

102
wsagent/che-core-api-project/src/test/java/org/eclipse/che/api/search/server/impl/FSLuceneSearcherTest.java Normal file
View File

@ -0,0 +1,102 @@
/*
* Copyright (c) 2012-2018 Red Hat, Inc.
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at
* http://www.eclipse.org/legal/epl-v10.html
*
* Contributors:
* Red Hat, Inc. - initial API and implementation
*/
package org.eclipse.che.api.search.server.impl;
/*
* Copyright (c) 2012-2017 Red Hat, Inc.
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at
* http://www.eclipse.org/legal/epl-v10.html
*
* Contributors:
* Red Hat, Inc. - initial API and implementation
*/
import static com.google.common.collect.Lists.newArrayList;
import static org.eclipse.che.api.search.SearcherTest.TEST_CONTENT;
import static org.testng.Assert.assertEquals;
import com.google.common.io.Files;
import java.io.File;
import java.nio.file.PathMatcher;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import org.eclipse.che.api.fs.server.impl.RootAwarePathTransformer;
import org.eclipse.che.api.search.SearcherTest.ContentBuilder;
import org.eclipse.che.api.search.server.QueryExpression;
import org.eclipse.che.commons.lang.IoUtil;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
@SuppressWarnings("Duplicates")
public class FSLuceneSearcherTest {
File indexDirectory;
File workspaceStorage;
Set<PathMatcher> excludePatterns;
LuceneSearcher searcher;
RootAwarePathTransformer pathTransformer;
ContentBuilder contentBuilder;
@BeforeMethod
public void setUp() throws Exception {
indexDirectory = Files.createTempDir();
workspaceStorage = Files.createTempDir();
excludePatterns = new HashSet<>();
pathTransformer = new RootAwarePathTransformer(workspaceStorage);
searcher =
new LuceneSearcher(excludePatterns, indexDirectory, workspaceStorage, pathTransformer);
contentBuilder = new ContentBuilder(workspaceStorage.toPath());
}
@AfterMethod
public void tearDown() throws Exception {
IoUtil.deleteRecursive(indexDirectory);
IoUtil.deleteRecursive(workspaceStorage);
}
@Test
public void shouldBeAbleToInitializesIndexForExistedFiles() throws Exception {
// given
contentBuilder
.createFolder("folder")
.createFile("xxx.txt", TEST_CONTENT[2])
.createFile("zzz.txt", TEST_CONTENT[1]);
// when
searcher.initialize();
searcher.getInitialIndexingLatch().await();
// then
List<String> paths = searcher.search(new QueryExpression().setText("think")).getFilePaths();
assertEquals(newArrayList("/folder/zzz.txt"), paths);
}
@Test
public void shouldBeAbleToExcludesFilesFromIndexWithFilter() throws Exception {
// given
contentBuilder
.createFolder("folder")
.createFile("xxx.txt", TEST_CONTENT[2])
.createFile("yyy.txt", TEST_CONTENT[2])
.createFile("zzz.txt", TEST_CONTENT[2]);
excludePatterns.add(
it -> it.toFile().isFile() && "yyy.txt".equals(it.getFileName().toString()));
searcher.add(contentBuilder.getCurrentFolder());
// when
List<String> paths = searcher.search(new QueryExpression().setText("be")).getFilePaths();
// then
assertEquals(newArrayList("/folder/xxx.txt", "/folder/zzz.txt"), paths);
}
}