Skip to content

Commit d356a46

Browse files
GMishxGMishx
committed
feat(search): add sorting index for lucene search
Add new `string` type index which can be used for sorting for `text` type indexes. Signed-off-by: Gaurav Mishra <[email protected]>
1 parent ba5bc9a commit d356a46

File tree

6 files changed

+151
-24
lines changed

6 files changed

+151
-24
lines changed

‎backend/common/src/main/java/org/eclipse/sw360/common/utils/SearchUtils.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@ public class SearchUtils {
6161
" }" +
6262
" if (result.trim().length > 0) {" +
6363
" index('text', indexName, result.trim(), {'store': true});" +
64+
" index('string', indexName + '_sort', result.trim());" +
6465
" }" +
6566
" }";
6667
}

‎backend/common/src/main/java/org/eclipse/sw360/datahandler/db/ComponentSearchHandler.java‎

Lines changed: 22 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -14,10 +14,7 @@
1414
import org.apache.logging.log4j.LogManager;
1515
import org.apache.logging.log4j.Logger;
1616
import org.eclipse.sw360.datahandler.cloudantclient.DatabaseConnectorCloudant;
17-
import org.eclipse.sw360.datahandler.common.SW360Constants;
1817
import org.eclipse.sw360.datahandler.couchdb.lucene.NouveauLuceneAwareDatabaseConnector;
19-
import org.eclipse.sw360.datahandler.resourcelists.ResourceClassNotFoundException;
20-
import org.eclipse.sw360.datahandler.resourcelists.ResourceComparatorGenerator;
2118
import org.eclipse.sw360.datahandler.thrift.PaginationData;
2219
import org.eclipse.sw360.datahandler.thrift.components.Component;
2320
import org.eclipse.sw360.datahandler.thrift.components.ComponentSortColumn;
@@ -30,11 +27,9 @@
3027
import javax.annotation.Nonnull;
3128
import java.io.IOException;
3229
import java.util.Collections;
33-
import java.util.Comparator;
3430
import java.util.List;
3531
import java.util.Map;
3632
import java.util.Set;
37-
import java.util.stream.Collectors;
3833

3934
import static org.eclipse.sw360.common.utils.SearchUtils.OBJ_ARRAY_TO_STRING_INDEX;
4035
import static org.eclipse.sw360.datahandler.permissions.PermissionUtils.makePermission;
@@ -66,9 +61,11 @@ public class ComponentSearchHandler {
6661
" arrayToStringIndex(doc.mainLicenseIds, 'mainLicenseIds');" +
6762
" if(doc.componentType && typeof(doc.componentType) == 'string' && doc.componentType.length > 0) {" +
6863
" index('text', 'componentType', doc.componentType, {'store': true});" +
64+
" index('string', 'componentType_sort', doc.componentType);" +
6965
" }" +
7066
" if(doc.name && typeof(doc.name) == 'string' && doc.name.length > 0) {" +
7167
" index('text', 'name', doc.name, {'store': true});"+
68+
" index('string', 'name_sort', doc.name);"+
7269
" }" +
7370
" if(doc.createdBy && typeof(doc.createdBy) == 'string' && doc.createdBy.length > 0) {" +
7471
" index('text', 'createdBy', doc.createdBy, {'store': true});"+
@@ -103,10 +100,11 @@ public List<Component> search(String text, final Map<String, Set<String>> subQue
103100

104101
public Map<PaginationData, List<Component>> searchAccessibleComponents(String text, final Map<String,
105102
Set<String>> subQueryRestrictions, User user, @Nonnull PaginationData pageData) {
103+
String sortColumn = getSortColumnName(pageData);
106104
Map<PaginationData, List<Component>> resultComponentList = connector
107105
.searchViewWithRestrictions(Component.class,
108106
luceneSearchView.getIndexName(), text, subQueryRestrictions,
109-
pageData, null, pageData.isAscending());
107+
pageData, sortColumn, pageData.isAscending());
110108

111109
PaginationData respPageData = resultComponentList.keySet().iterator().next();
112110
List<Component> componentList = resultComponentList.values().iterator().next();
@@ -119,12 +117,29 @@ public Map<PaginationData, List<Component>> searchAccessibleComponents(String te
119117
}
120118

121119
public List<Component> searchWithAccessibility(String text, final Map<String, Set<String>> subQueryRestrictions,
122-
User user ){
120+
User user) {
123121
List<Component> resultComponentList = connector.searchViewWithRestrictions(Component.class,
124122
luceneSearchView.getIndexName(), text, subQueryRestrictions);
125123
for (Component component : resultComponentList) {
126124
makePermission(component, user).fillPermissionsInOther(component);
127125
}
128126
return resultComponentList;
129127
}
128+
129+
/**
130+
* Convert sort column number back to sorting column name. This function makes sure to use the string column (with
131+
* `_sort` suffix) for text indexes.
132+
* @param pageData Pagination Data from the request.
133+
* @return Sort column name. Defaults to createdOn
134+
*/
135+
private static @Nonnull String getSortColumnName(@Nonnull PaginationData pageData) {
136+
return switch (ComponentSortColumn.findByValue(pageData.getSortColumnNumber())) {
137+
case ComponentSortColumn.BY_NAME -> "name_sort";
138+
case ComponentSortColumn.BY_VENDOR -> "vendorNames_sort";
139+
case ComponentSortColumn.BY_MAINLICENSE -> "mainLicenseIds_sort";
140+
case ComponentSortColumn.BY_TYPE -> "componentType_sort";
141+
case null -> "createdOn";
142+
default -> "createdOn";
143+
};
144+
}
130145
}

‎backend/common/src/main/java/org/eclipse/sw360/datahandler/db/ProjectSearchHandler.java‎

Lines changed: 36 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -13,11 +13,8 @@
1313
import com.google.gson.Gson;
1414
import org.eclipse.sw360.datahandler.cloudantclient.DatabaseConnectorCloudant;
1515
import org.eclipse.sw360.datahandler.common.DatabaseSettings;
16-
import org.eclipse.sw360.datahandler.common.SW360Constants;
1716
import org.eclipse.sw360.datahandler.couchdb.lucene.NouveauLuceneAwareDatabaseConnector;
1817
import org.eclipse.sw360.datahandler.permissions.ProjectPermissions;
19-
import org.eclipse.sw360.datahandler.resourcelists.ResourceClassNotFoundException;
20-
import org.eclipse.sw360.datahandler.resourcelists.ResourceComparatorGenerator;
2118
import org.eclipse.sw360.datahandler.thrift.PaginationData;
2219
import org.eclipse.sw360.datahandler.thrift.projects.Project;
2320
import org.eclipse.sw360.datahandler.thrift.projects.ProjectSortColumn;
@@ -26,9 +23,9 @@
2623
import org.eclipse.sw360.nouveau.designdocument.NouveauIndexDesignDocument;
2724
import org.eclipse.sw360.nouveau.designdocument.NouveauIndexFunction;
2825

26+
import javax.annotation.Nonnull;
2927
import java.io.IOException;
3028
import java.util.Collections;
31-
import java.util.Comparator;
3229
import java.util.HashMap;
3330
import java.util.HashSet;
3431
import java.util.List;
@@ -55,18 +52,26 @@ public class ProjectSearchHandler {
5552
" }" +
5653
" if(doc.projectType !== undefined && doc.projectType != null && doc.projectType.length >0) {" +
5754
" index('text', 'projectType', doc.projectType, {'store': true});" +
55+
" index('string', 'projectType_sort', doc.projectType);" +
5856
" }" +
5957
" if(doc.projectResponsible !== undefined && doc.projectResponsible != null && doc.projectResponsible.length >0) {" +
6058
" index('text', 'projectResponsible', doc.projectResponsible, {'store': true});" +
59+
" index('string', 'projectResponsible_sort', doc.projectResponsible);" +
6160
" }" +
6261
" if(doc.name !== undefined && doc.name != null && doc.name.length >0) {" +
6362
" index('text', 'name', doc.name, {'store': true});" +
63+
" index('string', 'name_sort', doc.name);" +
64+
" }" +
65+
" if(doc.description !== undefined && doc.description != null && doc.description.length >0) {" +
66+
" index('text', 'description', doc.description, {'store': true});" +
67+
" index('string', 'description_sort', doc.description);" +
6468
" }" +
6569
" if(doc.version !== undefined && doc.version != null && doc.version.length >0) {" +
6670
" index('string', 'version', doc.version, {'store': true});" +
6771
" }" +
6872
" if(doc.state !== undefined && doc.state != null && doc.state.length >0) {" +
6973
" index('text', 'state', doc.state, {'store': true});" +
74+
" index('string', 'state_sort', doc.state);" +
7075
" }" +
7176
" if(doc.clearingState) {" +
7277
" index('text', 'clearingState', doc.clearingState, {'store': true});" +
@@ -78,6 +83,11 @@ public class ProjectSearchHandler {
7883
" if(doc.releaseRelationNetwork !== undefined && doc.releaseRelationNetwork != null && doc.releaseRelationNetwork.length > 0) {" +
7984
" index('text', 'releaseRelationNetwork', doc.releaseRelationNetwork, {'store': true});" +
8085
" }" +
86+
" if(doc.createdOn && doc.createdOn.length) {"+
87+
" var dt = new Date(doc.createdOn);"+
88+
" var formattedDt = `${dt.getFullYear()}${(dt.getMonth()+1).toString().padStart(2,'0')}${dt.getDate().toString().padStart(2,'0')}`;" +
89+
" index('double', 'createdOn', Number(formattedDt), {'store': true});"+
90+
" }" +
8191
"}")
8292
.setFieldAnalyzer(
8393
Map.of("version", "keyword")
@@ -99,10 +109,11 @@ public ProjectSearchHandler(Cloudant client, String dbName) throws IOException {
99109
}
100110

101111
public Map<PaginationData, List<Project>> search(String text, final Map<String, Set<String>> subQueryRestrictions, User user, PaginationData pageData) {
112+
String sortColumn = getSortColumnName(pageData);
102113
Map<PaginationData, List<Project>> resultProjectList = connector
103114
.searchViewWithRestrictions(Project.class,
104115
luceneSearchView.getIndexName(), text, subQueryRestrictions,
105-
pageData, null, pageData.isAscending());
116+
pageData, sortColumn, pageData.isAscending());
106117

107118
PaginationData respPageData = resultProjectList.keySet().iterator().next();
108119
List<Project> projectList = resultProjectList.values().iterator().next();
@@ -155,4 +166,24 @@ private static Map<String, Set<String>> getFilterMapForSetReleaseIds(Set<String>
155166
filterMap.put(Project._Fields.RELEASE_RELATION_NETWORK.getFieldName(), values);
156167
return filterMap;
157168
}
169+
170+
/**
171+
* Convert sort column number back to sorting column name. This function makes sure to use the string column (with
172+
* `_sort` suffix) for text indexes.
173+
* @param pageData Pagination Data from the request.
174+
* @return Sort column name. Defaults to name_sort
175+
*/
176+
private static @Nonnull String getSortColumnName(@Nonnull PaginationData pageData) {
177+
return switch (ProjectSortColumn.findByValue(pageData.getSortColumnNumber())) {
178+
case ProjectSortColumn.BY_CREATEDON -> "createdOn";
179+
// case ProjectSortColumn.BY_VENDOR -> "vendor_sort";
180+
// case ProjectSortColumn.BY_MAINLICENSE -> "license_sort";
181+
case ProjectSortColumn.BY_TYPE -> "projectType_sort";
182+
case ProjectSortColumn.BY_DESCRIPTION -> "description_sort";
183+
case ProjectSortColumn.BY_RESPONSIBLE -> "projectResponsible_sort";
184+
case ProjectSortColumn.BY_STATE -> "state_sort";
185+
case null -> "name_sort";
186+
default -> "name_sort";
187+
};
188+
}
158189
}

‎backend/common/src/main/java/org/eclipse/sw360/datahandler/db/ReleaseSearchHandler.java‎

Lines changed: 26 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,10 +15,12 @@
1515
import org.eclipse.sw360.datahandler.couchdb.lucene.NouveauLuceneAwareDatabaseConnector;
1616
import org.eclipse.sw360.datahandler.thrift.PaginationData;
1717
import org.eclipse.sw360.datahandler.thrift.components.Release;
18+
import org.eclipse.sw360.datahandler.thrift.components.ReleaseSortColumn;
1819
import org.eclipse.sw360.nouveau.designdocument.NouveauDesignDocument;
1920
import org.eclipse.sw360.nouveau.designdocument.NouveauIndexDesignDocument;
2021
import org.eclipse.sw360.nouveau.designdocument.NouveauIndexFunction;
2122

23+
import javax.annotation.Nonnull;
2224
import java.io.IOException;
2325
import java.util.Collections;
2426
import java.util.List;
@@ -43,9 +45,16 @@ public class ReleaseSearchHandler {
4345
" if(doc.type == 'release') {" +
4446
" if (doc.name && typeof(doc.name) == 'string' && doc.name.length > 0) {" +
4547
" index('text', 'name', doc.name, {'store': true});" +
48+
" index('string', 'name_sort', doc.name);" +
4649
" }" +
4750
" if (doc.version && typeof(doc.version) == 'string' && doc.version.length > 0) {" +
4851
" index('text', 'version', doc.version, {'store': true});" +
52+
" index('string', 'version_sort', doc.version);" +
53+
" }" +
54+
" if(doc.createdOn && doc.createdOn.length) {"+
55+
" var dt = new Date(doc.createdOn);"+
56+
" var formattedDt = `${dt.getFullYear()}${(dt.getMonth()+1).toString().padStart(2,'0')}${dt.getDate().toString().padStart(2,'0')}`;" +
57+
" index('double', 'createdOn', Number(formattedDt), {'store': true});"+
4958
" }" +
5059
" index('text', 'id', doc._id, {'store': true});" +
5160
" }" +
@@ -64,17 +73,33 @@ public ReleaseSearchHandler(Cloudant cClient, String dbName) throws IOException
6473
}
6574

6675
public Map<PaginationData, List<Release>> search(String searchText, PaginationData pageData) {
76+
String sortColumn = getSortColumnName(pageData);
6777
Map<PaginationData, List<Release>> resultReleaseList = connector
6878
.searchViewWithRestrictions(Release.class,
6979
luceneSearchView.getIndexName(), null,
7080
Map.of(Release._Fields.NAME.getFieldName(),
7181
Collections.singleton(prepareWildcardQuery(searchText))
7282
),
73-
pageData, null, pageData.isAscending());
83+
pageData, sortColumn, pageData.isAscending());
7484

7585
PaginationData respPageData = resultReleaseList.keySet().iterator().next();
7686
List<Release> releaseList = resultReleaseList.values().iterator().next();
7787

7888
return Collections.singletonMap(respPageData, releaseList);
7989
}
90+
91+
/**
92+
* Convert sort column number back to sorting column name. This function makes sure to use the string column (with
93+
* `_sort` suffix) for text indexes.
94+
* @param pageData Pagination Data from the request.
95+
* @return Sort column name. Defaults to createdOn
96+
*/
97+
private static @Nonnull String getSortColumnName(@Nonnull PaginationData pageData) {
98+
return switch (ReleaseSortColumn.findByValue(pageData.getSortColumnNumber())) {
99+
case ReleaseSortColumn.BY_NAME -> "name_sort";
100+
case ReleaseSortColumn.BY_VERSION -> "version_sort";
101+
case null -> "createdOn";
102+
default -> "createdOn";
103+
};
104+
}
80105
}

‎backend/common/src/main/java/org/eclipse/sw360/datahandler/db/UserSearchHandler.java‎

Lines changed: 30 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -12,10 +12,7 @@
1212
import com.ibm.cloud.cloudant.v1.Cloudant;
1313
import com.google.gson.Gson;
1414
import org.eclipse.sw360.datahandler.cloudantclient.DatabaseConnectorCloudant;
15-
import org.eclipse.sw360.datahandler.common.SW360Constants;
1615
import org.eclipse.sw360.datahandler.couchdb.lucene.NouveauLuceneAwareDatabaseConnector;
17-
import org.eclipse.sw360.datahandler.resourcelists.ResourceClassNotFoundException;
18-
import org.eclipse.sw360.datahandler.resourcelists.ResourceComparatorGenerator;
1916
import org.eclipse.sw360.datahandler.thrift.PaginationData;
2017
import org.eclipse.sw360.datahandler.thrift.users.User;
2118
import org.eclipse.sw360.datahandler.thrift.users.UserSortColumn;
@@ -25,12 +22,11 @@
2522

2623
import javax.annotation.Nonnull;
2724
import java.io.IOException;
28-
import java.util.Collections;
29-
import java.util.Comparator;
3025
import java.util.List;
3126
import java.util.Map;
3227
import java.util.Set;
3328

29+
import static org.eclipse.sw360.common.utils.SearchUtils.OBJ_ARRAY_TO_STRING_INDEX;
3430
import static org.eclipse.sw360.datahandler.couchdb.lucene.NouveauLuceneAwareDatabaseConnector.prepareFuzzyQuery;
3531
import static org.eclipse.sw360.nouveau.LuceneAwareCouchDbConnector.DEFAULT_DESIGN_PREFIX;
3632

@@ -57,22 +53,31 @@ public class UserSearchHandler {
5753
private static final NouveauIndexDesignDocument luceneUserSearchView
5854
= new NouveauIndexDesignDocument("usersearch",
5955
new NouveauIndexFunction("function(doc) {" +
56+
OBJ_ARRAY_TO_STRING_INDEX +
6057
" if (!doc.type || doc.type != 'user') return;" +
6158
" if (doc.givenname && typeof(doc.givenname) == 'string' && doc.givenname.length > 0) {" +
6259
" index('text', 'givenname', doc.givenname, {'store': true});" +
60+
" index('string', 'givenname_sort', doc.givenname);" +
6361
" }" +
6462
" if (doc.lastname && typeof(doc.lastname) == 'string' && doc.lastname.length > 0) {" +
6563
" index('text', 'lastname', doc.lastname, {'store': true});" +
64+
" index('string', 'lastname_sort', doc.lastname);" +
6665
" }" +
6766
" if (doc.email && typeof(doc.email) == 'string' && doc.email.length > 0) {" +
6867
" index('text', 'email', doc.email, {'store': true});" +
68+
" index('string', 'email_sort', doc.email);" +
6969
" }" +
7070
" if (doc.userGroup && typeof(doc.userGroup) == 'string' && doc.userGroup.length > 0) {" +
7171
" index('text', 'userGroup', doc.userGroup, {'store': true});" +
7272
" }" +
7373
" if (doc.department && typeof(doc.department) == 'string' && doc.department.length > 0) {" +
7474
" index('text', 'department', doc.department, {'store': true});" +
75+
" index('string', 'department_sort', doc.department);" +
7576
" }" +
77+
" if (doc.deactivated && typeof(doc.deactivated) == 'boolean') {" +
78+
" index('double', 'deactivated', doc.deactivated ? 0 : 1);" +
79+
" }" +
80+
" arrayToStringIndex(doc.primaryRoles, 'primaryroles');" +
7681
"}"));
7782

7883
private final NouveauLuceneAwareDatabaseConnector connector;
@@ -105,8 +110,27 @@ public List<User> searchByNameAndEmail(String searchText) {
105110
}
106111

107112
public Map<PaginationData, List<User>> search(String text, final Map<String, Set<String>> subQueryRestrictions, @Nonnull PaginationData pageData) {
113+
String sortColumn = getSortColumnName(pageData);
108114
return connector.searchViewWithRestrictions(User.class,
109115
luceneUserSearchView.getIndexName(), text, subQueryRestrictions,
110-
pageData, null, pageData.isAscending());
116+
pageData, sortColumn, pageData.isAscending());
117+
}
118+
119+
/**
120+
* Convert sort column number back to sorting column name. This function makes sure to use the string column (with
121+
* `_sort` suffix) for text indexes.
122+
* @param pageData Pagination Data from the request.
123+
* @return Sort column name. Defaults to givenname_sort
124+
*/
125+
private static @Nonnull String getSortColumnName(@Nonnull PaginationData pageData) {
126+
return switch (UserSortColumn.findByValue(pageData.getSortColumnNumber())) {
127+
case UserSortColumn.BY_LASTNAME -> "lastname_sort";
128+
case UserSortColumn.BY_EMAIL -> "email_sort";
129+
case UserSortColumn.BY_STATUS -> "deactivated";
130+
case UserSortColumn.BY_DEPARTMENT -> "department_sort";
131+
case UserSortColumn.BY_ROLE -> "primaryroles_sort";
132+
case null -> "givenname_sort";
133+
default -> "givenname_sort";
134+
};
111135
}
112136
}

0 commit comments

Comments
 (0)