donnerpeter commented on code in PR #975:
URL: https://github.com/apache/lucene/pull/975#discussion_r906637207
##########
lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/WordFormGenerator.java:
##########
@@ -280,4 +300,188 @@ private boolean
isCompatibleWithPreviousAffixes(AffixedWord stem, Dictionary dic
return true;
}
}
+
+ private class WordCompressor {
+ private final Set<String> forbidden;
+ private final Runnable checkCanceled;
+ private final Set<String> wordSet;
+ private final Set<String> existingStems;
+ private final Map<String, Set<FlagSet>> stemToPossibleFlags = new
HashMap<>();
+ private final Map<String, Integer> stemCounts = new LinkedHashMap<>();
+
+ WordCompressor(List<String> words, Set<String> forbidden, Runnable
checkCanceled) {
+ this.forbidden = forbidden;
+ this.checkCanceled = checkCanceled;
+ wordSet = new HashSet<>(words);
+
+ Stemmer.StemCandidateProcessor processor =
+ new Stemmer.StemCandidateProcessor(WordContext.SIMPLE_WORD) {
+ @Override
+ boolean processStemCandidate(
+ char[] word,
+ int offset,
+ int length,
+ int lastAffix,
+ int outerPrefix,
+ int innerPrefix,
+ int outerSuffix,
+ int innerSuffix) {
+ String candidate = new String(word, offset, length);
+ stemCounts.merge(candidate, 1, Integer::sum);
+ Set<Character> flags = new LinkedHashSet<>();
+ if (outerPrefix >= 0)
flags.add(dictionary.affixData(outerPrefix, AFFIX_FLAG));
+ if (innerPrefix >= 0)
flags.add(dictionary.affixData(innerPrefix, AFFIX_FLAG));
+ if (outerSuffix >= 0)
flags.add(dictionary.affixData(outerSuffix, AFFIX_FLAG));
+ if (innerSuffix >= 0)
flags.add(dictionary.affixData(innerSuffix, AFFIX_FLAG));
+ stemToPossibleFlags
+ .computeIfAbsent(candidate, __ -> new LinkedHashSet<>())
+ .add(new FlagSet(flags, dictionary));
+ return true;
+ }
+ };
+
+ for (String word : words) {
+ checkCanceled.run();
+ stemCounts.merge(word, 1, Integer::sum);
+ stemToPossibleFlags.computeIfAbsent(word, __ -> new LinkedHashSet<>());
+ stemmer.removeAffixes(word.toCharArray(), 0, word.length(), true, -1,
-1, -1, processor);
+ }
+
+ existingStems =
+ stemCounts.keySet().stream()
+ .filter(stem -> dictionary.lookupEntries(stem) != null)
+ .collect(Collectors.toSet());
+ }
+
+ EntrySuggestion compress() {
+ Comparator<String> stemSorter =
+ Comparator.comparing((String s) -> existingStems.contains(s))
+ .thenComparing(stemCounts::get)
+ .reversed();
+ List<String> sortedStems =
stemCounts.keySet().stream().sorted(stemSorter).toList();
+ PriorityQueue<State> queue = new PriorityQueue<>();
+ queue.offer(new State(Map.of(), wordSet.size(), 0, 0));
+ State result = null;
+ while (!queue.isEmpty()) {
+ State state = queue.poll();
+ if (state.underGenerated == 0) {
+ if (result == null || state.compareTo(result) < 0) result = state;
+ if (state.forbidden == 0) break;
+ continue;
+ }
+
+ for (String stem : sortedStems) {
+ if (!state.stemToFlags.containsKey(stem)) {
+ queue.offer(addStem(state, stem));
+ }
+ }
+
+ for (Map.Entry<String, Set<FlagSet>> entry :
state.stemToFlags.entrySet()) {
+ for (FlagSet flags : stemToPossibleFlags.get(entry.getKey())) {
+ if (!entry.getValue().contains(flags)) {
+ queue.offer(addFlags(state, entry.getKey(), flags));
+ }
+ }
+ }
+ }
+ return result == null ? null : toSuggestion(result);
+ }
+
+ EntrySuggestion toSuggestion(State state) {
+ List<DictEntry> toEdit = new ArrayList<>();
+ List<DictEntry> toAdd = new ArrayList<>();
+ for (Map.Entry<String, Set<FlagSet>> entry :
state.stemToFlags.entrySet()) {
+ addEntry(toEdit, toAdd, entry.getKey(),
FlagSet.flatten(entry.getValue()));
+ }
+
+ List<String> extraGenerated = new ArrayList<>();
+ for (String extra :
allGenerated(state.stemToFlags).distinct().sorted().toList()) {
+ if (wordSet.contains(extra)) continue;
+
+ if (forbidden.contains(extra) && dictionary.forbiddenword !=
FLAG_UNSET) {
+ addEntry(toEdit, toAdd, extra, Set.of(dictionary.forbiddenword));
+ } else {
+ extraGenerated.add(extra);
+ }
+ }
+
+ return new EntrySuggestion(toEdit, toAdd, extraGenerated);
+ }
+
+ private void addEntry(
+ List<DictEntry> toEdit, List<DictEntry> toAdd, String stem,
Set<Character> flags) {
+ String flagString = toFlagString(flags);
+ (existingStems.contains(stem) ? toEdit :
toAdd).add(DictEntry.create(stem, flagString));
+ }
+
+ private State addStem(State state, String stem) {
+ LinkedHashMap<String, Set<FlagSet>> stemToFlags = new
LinkedHashMap<>(state.stemToFlags);
+ stemToFlags.put(stem, Set.of());
+ return newState(stemToFlags);
+ }
+
+ private State addFlags(State state, String stem, FlagSet flags) {
+ LinkedHashMap<String, Set<FlagSet>> stemToFlags = new
LinkedHashMap<>(state.stemToFlags);
+ Set<FlagSet> flagSets = new LinkedHashSet<>(stemToFlags.get(stem));
+ flagSets.add(flags);
+ stemToFlags.put(stem, flagSets);
+ return newState(stemToFlags);
+ }
+
+ private State newState(Map<String, Set<FlagSet>> stemToFlags) {
+ Set<String> allGenerated =
allGenerated(stemToFlags).collect(Collectors.toSet());
+ return new State(
+ stemToFlags,
+ (int) wordSet.stream().filter(s ->
!allGenerated.contains(s)).count(),
+ (int) allGenerated.stream().filter(s ->
!wordSet.contains(s)).count(),
+ (int) allGenerated.stream().filter(s ->
forbidden.contains(s)).count());
+ }
+
+ private final Map<StemWithFlags, List<String>> expansionCache = new
HashMap<>();
+
+ private record StemWithFlags(String stem, Set<FlagSet> flags) {}
+
+ private Stream<String> allGenerated(Map<String, Set<FlagSet>> stemToFlags)
{
+ Function<StemWithFlags, List<String>> expandToWords =
+ e -> expand(e.stem, FlagSet.flatten(e.flags)).stream().map(w ->
w.getWord()).toList();
+ return stemToFlags.entrySet().stream()
+ .map(e -> new StemWithFlags(e.getKey(), e.getValue()))
+ .flatMap(swc -> expansionCache.computeIfAbsent(swc,
expandToWords).stream());
+ }
+
+ private List<AffixedWord> expand(String stem, Set<Character> flagSet) {
+ return expandRoot(stem, toFlagString(flagSet), checkCanceled);
+ }
+
+ private String toFlagString(Set<Character> flagSet) {
+ return
dictionary.flagParsingStrategy.printFlags(Dictionary.toSortedCharArray(flagSet));
+ }
+ }
+
+ private record FlagSet(Set<Character> flags, Dictionary dictionary) {
+ static Set<Character> flatten(Set<FlagSet> flagSets) {
+ return flagSets.stream().flatMap(f ->
f.flags.stream()).collect(Collectors.toSet());
+ }
+
+ @Override
+ public String toString() {
+ return
dictionary.flagParsingStrategy.printFlags(Dictionary.toSortedCharArray(flags));
+ }
+ }
+
+ private record State(
Review Comment:
Makes sense, thanks!
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]