Skip to content

Commit

Permalink
fix(search): Detect field type for use in defining the sort order (#8992
Browse files Browse the repository at this point in the history 
)

Co-authored-by: Indy Prentice <indy@Indys-MacBook-Pro.local>
  • Loading branch information
@iprentic
iprentic and Indy Prentice authored Oct 18, 2023
1 parent 1b73724 commit aae1347
Show file tree
Hide file tree
Showing 4 changed files with 154 additions and 40 deletions.
48 changes: 20 additions & 28 deletions ...rc/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/MappingsBuilder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import com.linkedin.metadata.models.SearchScoreFieldSpec;
import com.linkedin.metadata.models.SearchableFieldSpec;
import com.linkedin.metadata.models.annotation.SearchableAnnotation.FieldType;
import com.linkedin.metadata.search.utils.ESUtils;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
Expand All @@ -31,15 +32,6 @@ public static Map<String, String> getPartialNgramConfigWithOverrides(Map<String,

public static final Map<String, String> KEYWORD_TYPE_MAP = ImmutableMap.of(TYPE, KEYWORD);

// Field Types
public static final String BOOLEAN = "boolean";
public static final String DATE = "date";
public static final String DOUBLE = "double";
public static final String LONG = "long";
public static final String OBJECT = "object";
public static final String TEXT = "text";
public static final String TOKEN_COUNT = "token_count";

// Subfields
public static final String DELIMITED = "delimited";
public static final String LENGTH = "length";
Expand Down Expand Up @@ -74,7 +66,7 @@ public static Map<String, Object> getMappings(@Nonnull final EntitySpec entitySp
private static Map<String, Object> getMappingsForUrn() {
Map<String, Object> subFields = new HashMap<>();
subFields.put(DELIMITED, ImmutableMap.of(
TYPE, TEXT,
TYPE, ESUtils.TEXT_FIELD_TYPE,
ANALYZER, URN_ANALYZER,
SEARCH_ANALYZER, URN_SEARCH_ANALYZER,
SEARCH_QUOTE_ANALYZER, CUSTOM_QUOTE_ANALYZER)
Expand All @@ -85,13 +77,13 @@ private static Map<String, Object> getMappingsForUrn() {
)
));
return ImmutableMap.<String, Object>builder()
.put(TYPE, KEYWORD)
.put(TYPE, ESUtils.KEYWORD_FIELD_TYPE)
.put(FIELDS, subFields)
.build();
}

private static Map<String, Object> getMappingsForRunId() {
return ImmutableMap.<String, Object>builder().put(TYPE, KEYWORD).build();
return ImmutableMap.<String, Object>builder().put(TYPE, ESUtils.KEYWORD_FIELD_TYPE).build();
}

private static Map<String, Object> getMappingsForField(@Nonnull final SearchableFieldSpec searchableFieldSpec) {
Expand All @@ -104,23 +96,23 @@ private static Map<String, Object> getMappingsForField(@Nonnull final Searchable
} else if (fieldType == FieldType.TEXT || fieldType == FieldType.TEXT_PARTIAL || fieldType == FieldType.WORD_GRAM) {
mappingForField.putAll(getMappingsForSearchText(fieldType));
} else if (fieldType == FieldType.BROWSE_PATH) {
mappingForField.put(TYPE, TEXT);
mappingForField.put(TYPE, ESUtils.TEXT_FIELD_TYPE);
mappingForField.put(FIELDS,
ImmutableMap.of(LENGTH, ImmutableMap.of(
TYPE, TOKEN_COUNT,
TYPE, ESUtils.TOKEN_COUNT_FIELD_TYPE,
ANALYZER, SLASH_PATTERN_ANALYZER)));
mappingForField.put(ANALYZER, BROWSE_PATH_HIERARCHY_ANALYZER);
mappingForField.put(FIELDDATA, true);
} else if (fieldType == FieldType.BROWSE_PATH_V2) {
mappingForField.put(TYPE, TEXT);
mappingForField.put(TYPE, ESUtils.TEXT_FIELD_TYPE);
mappingForField.put(FIELDS,
ImmutableMap.of(LENGTH, ImmutableMap.of(
TYPE, TOKEN_COUNT,
TYPE, ESUtils.TOKEN_COUNT_FIELD_TYPE,
ANALYZER, UNIT_SEPARATOR_PATTERN_ANALYZER)));
mappingForField.put(ANALYZER, BROWSE_PATH_V2_HIERARCHY_ANALYZER);
mappingForField.put(FIELDDATA, true);
} else if (fieldType == FieldType.URN || fieldType == FieldType.URN_PARTIAL) {
mappingForField.put(TYPE, TEXT);
mappingForField.put(TYPE, ESUtils.TEXT_FIELD_TYPE);
mappingForField.put(ANALYZER, URN_ANALYZER);
mappingForField.put(SEARCH_ANALYZER, URN_SEARCH_ANALYZER);
mappingForField.put(SEARCH_QUOTE_ANALYZER, CUSTOM_QUOTE_ANALYZER);
Expand All @@ -135,32 +127,32 @@ private static Map<String, Object> getMappingsForField(@Nonnull final Searchable
subFields.put(KEYWORD, KEYWORD_TYPE_MAP);
mappingForField.put(FIELDS, subFields);
} else if (fieldType == FieldType.BOOLEAN) {
mappingForField.put(TYPE, BOOLEAN);
mappingForField.put(TYPE, ESUtils.BOOLEAN_FIELD_TYPE);
} else if (fieldType == FieldType.COUNT) {
mappingForField.put(TYPE, LONG);
mappingForField.put(TYPE, ESUtils.LONG_FIELD_TYPE);
} else if (fieldType == FieldType.DATETIME) {
mappingForField.put(TYPE, DATE);
mappingForField.put(TYPE, ESUtils.DATE_FIELD_TYPE);
} else if (fieldType == FieldType.OBJECT) {
mappingForField.put(TYPE, OBJECT);
mappingForField.put(TYPE, ESUtils.DATE_FIELD_TYPE);
} else {
log.info("FieldType {} has no mappings implemented", fieldType);
}
mappings.put(searchableFieldSpec.getSearchableAnnotation().getFieldName(), mappingForField);

searchableFieldSpec.getSearchableAnnotation()
.getHasValuesFieldName()
.ifPresent(fieldName -> mappings.put(fieldName, ImmutableMap.of(TYPE, BOOLEAN)));
.ifPresent(fieldName -> mappings.put(fieldName, ImmutableMap.of(TYPE, ESUtils.BOOLEAN_FIELD_TYPE)));
searchableFieldSpec.getSearchableAnnotation()
.getNumValuesFieldName()
.ifPresent(fieldName -> mappings.put(fieldName, ImmutableMap.of(TYPE, LONG)));
.ifPresent(fieldName -> mappings.put(fieldName, ImmutableMap.of(TYPE, ESUtils.LONG_FIELD_TYPE)));
mappings.putAll(getMappingsForFieldNameAliases(searchableFieldSpec));

return mappings;
}

private static Map<String, Object> getMappingsForKeyword() {
Map<String, Object> mappingForField = new HashMap<>();
mappingForField.put(TYPE, KEYWORD);
mappingForField.put(TYPE, ESUtils.KEYWORD_FIELD_TYPE);
mappingForField.put(NORMALIZER, KEYWORD_NORMALIZER);
// Add keyword subfield without lowercase filter
mappingForField.put(FIELDS, ImmutableMap.of(KEYWORD, KEYWORD_TYPE_MAP));
Expand All @@ -169,7 +161,7 @@ private static Map<String, Object> getMappingsForKeyword() {

private static Map<String, Object> getMappingsForSearchText(FieldType fieldType) {
Map<String, Object> mappingForField = new HashMap<>();
mappingForField.put(TYPE, KEYWORD);
mappingForField.put(TYPE, ESUtils.KEYWORD_FIELD_TYPE);
mappingForField.put(NORMALIZER, KEYWORD_NORMALIZER);
Map<String, Object> subFields = new HashMap<>();
if (fieldType == FieldType.TEXT_PARTIAL || fieldType == FieldType.WORD_GRAM) {
Expand All @@ -186,14 +178,14 @@ private static Map<String, Object> getMappingsForSearchText(FieldType fieldType)
String fieldName = entry.getKey();
String analyzerName = entry.getValue();
subFields.put(fieldName, ImmutableMap.of(
TYPE, TEXT,
TYPE, ESUtils.TEXT_FIELD_TYPE,
ANALYZER, analyzerName
));
}
}
}
subFields.put(DELIMITED, ImmutableMap.of(
TYPE, TEXT,
TYPE, ESUtils.TEXT_FIELD_TYPE,
ANALYZER, TEXT_ANALYZER,
SEARCH_ANALYZER, TEXT_SEARCH_ANALYZER,
SEARCH_QUOTE_ANALYZER, CUSTOM_QUOTE_ANALYZER));
Expand All @@ -206,7 +198,7 @@ private static Map<String, Object> getMappingsForSearchText(FieldType fieldType)
private static Map<String, Object> getMappingsForSearchScoreField(
@Nonnull final SearchScoreFieldSpec searchScoreFieldSpec) {
return ImmutableMap.of(searchScoreFieldSpec.getSearchScoreAnnotation().getFieldName(),
ImmutableMap.of(TYPE, DOUBLE));
ImmutableMap.of(TYPE, ESUtils.DOUBLE_FIELD_TYPE));
}

private static Map<String, Object> getMappingsForFieldNameAliases(@Nonnull final SearchableFieldSpec searchableFieldSpec) {
Expand Down
8 changes: 4 additions & 4 deletions ...n/java/com/linkedin/metadata/search/elasticsearch/query/request/SearchRequestHandler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -202,7 +202,7 @@ public SearchRequest getSearchRequest(@Nonnull String input, @Nullable Filter fi
if (!finalSearchFlags.isSkipHighlighting()) {
searchSourceBuilder.highlighter(_highlights);
}
ESUtils.buildSortOrder(searchSourceBuilder, sortCriterion);
ESUtils.buildSortOrder(searchSourceBuilder, sortCriterion, _entitySpecs);

if (finalSearchFlags.isGetSuggestions()) {
ESUtils.buildNameSuggestions(searchSourceBuilder, input);
Expand Down Expand Up @@ -243,7 +243,7 @@ public SearchRequest getSearchRequest(@Nonnull String input, @Nullable Filter fi
searchSourceBuilder.query(QueryBuilders.boolQuery().must(getQuery(input, finalSearchFlags.isFulltext())).filter(filterQuery));
_aggregationQueryBuilder.getAggregations().forEach(searchSourceBuilder::aggregation);
searchSourceBuilder.highlighter(getHighlights());
ESUtils.buildSortOrder(searchSourceBuilder, sortCriterion);
ESUtils.buildSortOrder(searchSourceBuilder, sortCriterion, _entitySpecs);
searchRequest.source(searchSourceBuilder);
log.debug("Search request is: " + searchRequest);
searchRequest.indicesOptions(null);
Expand All @@ -270,7 +270,7 @@ public SearchRequest getFilterRequest(@Nullable Filter filters, @Nullable SortCr
final SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(filterQuery);
searchSourceBuilder.from(from).size(size);
ESUtils.buildSortOrder(searchSourceBuilder, sortCriterion);
ESUtils.buildSortOrder(searchSourceBuilder, sortCriterion, _entitySpecs);
searchRequest.source(searchSourceBuilder);

return searchRequest;
Expand Down Expand Up @@ -301,7 +301,7 @@ public SearchRequest getFilterRequest(@Nullable Filter filters, @Nullable SortCr
searchSourceBuilder.size(size);

ESUtils.setSearchAfter(searchSourceBuilder, sort, pitId, keepAlive);
ESUtils.buildSortOrder(searchSourceBuilder, sortCriterion);
ESUtils.buildSortOrder(searchSourceBuilder, sortCriterion, _entitySpecs);
searchRequest.source(searchSourceBuilder);

return searchRequest;
Expand Down
74 changes: 71 additions & 3 deletions metadata-io/src/main/java/com/linkedin/metadata/search/utils/ESUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,9 @@

import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.linkedin.metadata.models.EntitySpec;
import com.linkedin.metadata.models.SearchableFieldSpec;
import com.linkedin.metadata.models.annotation.SearchableAnnotation;
import com.linkedin.metadata.query.filter.Condition;
import com.linkedin.metadata.query.filter.ConjunctiveCriterion;
import com.linkedin.metadata.query.filter.Criterion;
Expand Down Expand Up @@ -49,7 +52,28 @@ public class ESUtils {
public static final int MAX_RESULT_SIZE = 10000;
public static final String OPAQUE_ID_HEADER = "X-Opaque-Id";
public static final String HEADER_VALUE_DELIMITER = "|";
public static final String KEYWORD_TYPE = "keyword";

// Field types
public static final String KEYWORD_FIELD_TYPE = "keyword";
public static final String BOOLEAN_FIELD_TYPE = "boolean";
public static final String DATE_FIELD_TYPE = "date";
public static final String DOUBLE_FIELD_TYPE = "double";
public static final String LONG_FIELD_TYPE = "long";
public static final String OBJECT_FIELD_TYPE = "object";
public static final String TEXT_FIELD_TYPE = "text";
public static final String TOKEN_COUNT_FIELD_TYPE = "token_count";
// End of field types

public static final Set<SearchableAnnotation.FieldType> FIELD_TYPES_STORED_AS_KEYWORD = Set.of(
SearchableAnnotation.FieldType.KEYWORD,
SearchableAnnotation.FieldType.TEXT,
SearchableAnnotation.FieldType.TEXT_PARTIAL,
SearchableAnnotation.FieldType.WORD_GRAM);
public static final Set<SearchableAnnotation.FieldType> FIELD_TYPES_STORED_AS_TEXT = Set.of(
SearchableAnnotation.FieldType.BROWSE_PATH,
SearchableAnnotation.FieldType.BROWSE_PATH_V2,
SearchableAnnotation.FieldType.URN,
SearchableAnnotation.FieldType.URN_PARTIAL);
public static final String ENTITY_NAME_FIELD = "_entityName";
public static final String NAME_SUGGESTION = "nameSuggestion";

Expand Down Expand Up @@ -174,6 +198,25 @@ public static QueryBuilder getQueryBuilderFromCriterion(@Nonnull final Criterion
return getQueryBuilderFromCriterionForSingleField(criterion, isTimeseries);
}

public static String getElasticTypeForFieldType(SearchableAnnotation.FieldType fieldType) {
if (FIELD_TYPES_STORED_AS_KEYWORD.contains(fieldType)) {
return KEYWORD_FIELD_TYPE;
} else if (FIELD_TYPES_STORED_AS_TEXT.contains(fieldType)) {
return TEXT_FIELD_TYPE;
} else if (fieldType == SearchableAnnotation.FieldType.BOOLEAN) {
return BOOLEAN_FIELD_TYPE;
} else if (fieldType == SearchableAnnotation.FieldType.COUNT) {
return LONG_FIELD_TYPE;
} else if (fieldType == SearchableAnnotation.FieldType.DATETIME) {
return DATE_FIELD_TYPE;
} else if (fieldType == SearchableAnnotation.FieldType.OBJECT) {
return OBJECT_FIELD_TYPE;
} else {
log.warn("FieldType {} has no mappings implemented", fieldType);
return null;
}
}

/**
* Populates source field of search query with the sort order as per the criterion provided.
*
Expand All @@ -189,14 +232,39 @@ public static QueryBuilder getQueryBuilderFromCriterion(@Nonnull final Criterion
* @param sortCriterion {@link SortCriterion} to be applied to the search results
*/
public static void buildSortOrder(@Nonnull SearchSourceBuilder searchSourceBuilder,
@Nullable SortCriterion sortCriterion) {
@Nullable SortCriterion sortCriterion, List<EntitySpec> entitySpecs) {
if (sortCriterion == null) {
searchSourceBuilder.sort(new ScoreSortBuilder().order(SortOrder.DESC));
} else {
Optional<SearchableAnnotation.FieldType> fieldTypeForDefault = Optional.empty();
for (EntitySpec entitySpec : entitySpecs) {
List<SearchableFieldSpec> fieldSpecs = entitySpec.getSearchableFieldSpecs();
for (SearchableFieldSpec fieldSpec : fieldSpecs) {
SearchableAnnotation annotation = fieldSpec.getSearchableAnnotation();
if (annotation.getFieldName().equals(sortCriterion.getField())
|| annotation.getFieldNameAliases().contains(sortCriterion.getField())) {
fieldTypeForDefault = Optional.of(fieldSpec.getSearchableAnnotation().getFieldType());
break;
}
}
if (fieldTypeForDefault.isPresent()) {
break;
}
}
if (fieldTypeForDefault.isEmpty()) {
log.warn("Sort criterion field " + sortCriterion.getField() + " was not found in any entity spec to be searched");
}
final SortOrder esSortOrder =
(sortCriterion.getOrder() == com.linkedin.metadata.query.filter.SortOrder.ASCENDING) ? SortOrder.ASC
: SortOrder.DESC;
searchSourceBuilder.sort(new FieldSortBuilder(sortCriterion.getField()).order(esSortOrder).unmappedType(KEYWORD_TYPE));
FieldSortBuilder sortBuilder = new FieldSortBuilder(sortCriterion.getField()).order(esSortOrder);
if (fieldTypeForDefault.isPresent()) {
String esFieldtype = getElasticTypeForFieldType(fieldTypeForDefault.get());
if (esFieldtype != null) {
sortBuilder.unmappedType(esFieldtype);
}
}
searchSourceBuilder.sort(sortBuilder);
}
if (sortCriterion == null || !sortCriterion.getField().equals(DEFAULT_SEARCH_RESULTS_SORT_BY_FIELD)) {
searchSourceBuilder.sort(new FieldSortBuilder(DEFAULT_SEARCH_RESULTS_SORT_BY_FIELD).order(SortOrder.ASC));
Expand Down
Loading

0 comments on commit aae1347

Please sign in to comment.