public inbox for [email protected]
help / color / mirror / Atom feed[PATCH 7/9] Remove the special batch mode, use a larger buffer always
19+ messages / 7 participants
[nested] [flat]
* [PATCH 7/9] Remove the special batch mode, use a larger buffer always
@ 2021-02-02 00:57 Tomas Vondra <[email protected]>
0 siblings, 0 replies; 19+ messages in thread
From: Tomas Vondra @ 2021-02-02 00:57 UTC (permalink / raw)
Instead of using a batch mode (with a larger input buffer) only for new
ranges, which introduces "special cases" in various places, use it as
the standard approach.
Also, instead of sizing the buffer to cover the whole range, limit it
to some reasonable limit (10x the user-specified size). That should give
us most of the benefits without consuming a lot of memory.
---
src/backend/access/brin/brin_minmax_multi.c | 851 ++++++++++++--------
1 file changed, 525 insertions(+), 326 deletions(-)
diff --git a/src/backend/access/brin/brin_minmax_multi.c b/src/backend/access/brin/brin_minmax_multi.c
index 69a72da337..08d0d55b06 100644
--- a/src/backend/access/brin/brin_minmax_multi.c
+++ b/src/backend/access/brin/brin_minmax_multi.c
@@ -92,7 +92,15 @@
*/
#define PROCNUM_BASE 11
-#define MINMAX_LOAD_FACTOR 0.75
+/*
+ * Sizing the insert buffer - we use 10x the number of values specified
+ * in the reloption, but we cap it to 8192 not to get too large. When
+ * the buffer gets full, we reduce the number of values by half.
+ */
+#define MINMAX_BUFFER_FACTOR 10
+#define MINMAX_BUFFER_MIN 256
+#define MINMAX_BUFFER_MAX 8192
+#define MINMAX_BUFFER_LOAD_FACTOR 0.5
typedef struct MinmaxMultiOpaque
{
@@ -155,23 +163,24 @@ typedef struct Ranges
Oid typid;
Oid colloid;
AttrNumber attno;
+ FmgrInfo *cmp;
/* (2*nranges + nvalues) <= maxvalues */
int nranges; /* number of ranges in the array (stored) */
+ int nsorted; /* number of sorted values (ranges + points) */
int nvalues; /* number of values in the data array (all) */
int maxvalues; /* maximum number of values (reloption) */
/*
- * In batch mode, we simply add the values into a buffer, without any
- * expensive steps (sorting, deduplication, ...). The buffer is sized
- * to be larger than the target number of values per range, which
- * reduces the number of compactions - operating on larger buffers is
- * significantly more efficient, in most cases. We keep the actual
- * target and compact to the requested number of values at the very
- * end, before serializing to on-disk representation.
+ * We simply add the values into a large buffer, without any expensive
+ * steps (sorting, deduplication, ...). The buffer is a multiple of
+ * the target number of values, so the compaction happen less often,
+ * amortizing the costs. We keep the actual target and compact to
+ * the requested number of values at the very end, before serializing
+ * to on-disk representation.
*/
- bool batch_mode;
- int target_maxvalues; /* requested number of values */
+ /* requested number of values */
+ int target_maxvalues;
/* values stored for this range - either raw values, or ranges */
Datum values[FLEXIBLE_ARRAY_MEMBER];
@@ -203,7 +212,7 @@ typedef struct SerializedRanges
static SerializedRanges *range_serialize(Ranges *range);
-static Ranges *range_deserialize(SerializedRanges *range);
+static Ranges *range_deserialize(int maxvalues, SerializedRanges *range);
/* Cache for support and strategy procesures. */
@@ -213,6 +222,14 @@ static FmgrInfo *minmax_multi_get_procinfo(BrinDesc *bdesc, uint16 attno,
static FmgrInfo *minmax_multi_get_strategy_procinfo(BrinDesc *bdesc,
uint16 attno, Oid subtype, uint16 strategynum);
+typedef struct compare_context
+{
+ FmgrInfo *cmpFn;
+ Oid colloid;
+} compare_context;
+
+static int compare_values(const void *a, const void *b, void *arg);
+
/*
* minmax_multi_init
@@ -240,6 +257,57 @@ minmax_multi_init(int maxvalues)
return ranges;
}
+static void
+AssertCheckRanges(Ranges *ranges, FmgrInfo *cmpFn, Oid colloid);
+
+
+static void
+range_deduplicate_values(Ranges *range)
+{
+ int i, n;
+ int start;
+ compare_context cxt;
+
+ /*
+ * If there are no unsorted values, we're done (this probably can't
+ * happen, as we're adding values to unsorted part).
+ */
+ if (range->nsorted == range->nvalues)
+ return;
+
+ /* sort the values */
+ cxt.colloid = range->colloid;
+ cxt.cmpFn = range->cmp;
+
+ /* how many values to sort? */
+ start = 2 * range->nranges;
+
+ qsort_arg(&range->values[start],
+ range->nvalues, sizeof(Datum),
+ compare_values, (void *) &cxt);
+
+ n = 1;
+ for (i = 1; i < range->nvalues; i++)
+ {
+ /* same as preceding value, so store it */
+ if (compare_values(&range->values[start + i - 1],
+ &range->values[start + i],
+ (void *) &cxt) == 0)
+ continue;
+
+ range->values[start + n] = range->values[start + i];
+
+ n++;
+ }
+
+ /* now all the values are sorted */
+ range->nvalues = n;
+ range->nsorted = n;
+
+ AssertCheckRanges(range, range->cmp, range->colloid);
+}
+
+
/*
* range_serialize
* Serialize the in-memory representation into a compact varlena value.
@@ -262,14 +330,25 @@ range_serialize(Ranges *range)
/* simple sanity checks */
Assert(range->nranges >= 0);
+ Assert(range->nsorted >= 0);
Assert(range->nvalues >= 0);
Assert(range->maxvalues > 0);
+ Assert(range->target_maxvalues > 0);
+
+ /* at this point the range should be compacted to the target size */
+ Assert(2*range->nranges + range->nvalues <= range->target_maxvalues);
+
+ Assert(range->target_maxvalues <= range->maxvalues);
+
+ /* range boundaries are always sorted */
+ Assert(range->nvalues >= range->nsorted);
+
+ /* sort and deduplicate values, if there's unsorted part */
+ range_deduplicate_values(range);
/* see how many Datum values we actually have */
nvalues = 2*range->nranges + range->nvalues;
- Assert(2*range->nranges + range->nvalues <= range->maxvalues);
-
typid = range->typid;
typbyval = get_typbyval(typid);
typlen = get_typlen(typid);
@@ -316,7 +395,7 @@ range_serialize(Ranges *range)
serialized->typid = typid;
serialized->nranges = range->nranges;
serialized->nvalues = range->nvalues;
- serialized->maxvalues = range->maxvalues;
+ serialized->maxvalues = range->target_maxvalues;
/*
* And now copy also the boundary values (like the length calculation
@@ -367,7 +446,7 @@ range_serialize(Ranges *range)
* in the in-memory value array.
*/
static Ranges *
-range_deserialize(SerializedRanges *serialized)
+range_deserialize(int maxvalues, SerializedRanges *serialized)
{
int i,
nvalues;
@@ -384,15 +463,18 @@ range_deserialize(SerializedRanges *serialized)
nvalues = 2*serialized->nranges + serialized->nvalues;
Assert(nvalues <= serialized->maxvalues);
+ Assert(serialized->maxvalues <= maxvalues);
- range = minmax_multi_init(serialized->maxvalues);
+ range = minmax_multi_init(maxvalues);
/* copy the header info */
range->nranges = serialized->nranges;
range->nvalues = serialized->nvalues;
- range->maxvalues = serialized->maxvalues;
+ range->nsorted = serialized->nvalues;
+ range->maxvalues = maxvalues;
+ range->target_maxvalues = serialized->maxvalues;
+
range->typid = serialized->typid;
- range->batch_mode = false;
typbyval = get_typbyval(serialized->typid);
typlen = get_typlen(serialized->typid);
@@ -439,12 +521,6 @@ range_deserialize(SerializedRanges *serialized)
return range;
}
-typedef struct compare_context
-{
- FmgrInfo *cmpFn;
- Oid colloid;
-} compare_context;
-
/*
* Used to represent ranges expanded during merging and combining (to
* reduce number of boundary values to store).
@@ -528,6 +604,115 @@ compare_values(const void *a, const void *b, void *arg)
return 0;
}
+void *bsearch_arg(const void *key, const void *base,
+ size_t nmemb, size_t size,
+ int (*compar) (const void *, const void *, void *),
+ void *arg);
+
+static bool
+has_matching_range(BrinDesc *bdesc, Oid colloid, Ranges *ranges,
+ Datum newval, AttrNumber attno, Oid typid)
+{
+ Datum compar;
+
+ Datum minvalue = ranges->values[0];
+ Datum maxvalue = ranges->values[2*ranges->nranges - 1];
+
+ FmgrInfo *cmpLessFn;
+ FmgrInfo *cmpGreaterFn;
+
+ /* binary search on ranges */
+ int start,
+ end;
+
+ if (ranges->nranges == 0)
+ return false;
+
+ /*
+ * Otherwise, need to compare the new value with boundaries of all
+ * the ranges. First check if it's less than the absolute minimum,
+ * which is the first value in the array.
+ */
+ cmpLessFn = minmax_multi_get_strategy_procinfo(bdesc, attno, typid,
+ BTLessStrategyNumber);
+ compar = FunctionCall2Coll(cmpLessFn, colloid, newval, minvalue);
+
+ /* smaller than the smallest value in the range list */
+ if (DatumGetBool(compar))
+ return false;
+
+ /*
+ * And now compare it to the existing maximum (last value in the
+ * data array). But only if we haven't already ruled out a possible
+ * match in the minvalue check.
+ */
+ cmpGreaterFn = minmax_multi_get_strategy_procinfo(bdesc, attno, typid,
+ BTGreaterStrategyNumber);
+ compar = FunctionCall2Coll(cmpGreaterFn, colloid, newval, maxvalue);
+
+ if (DatumGetBool(compar))
+ return false;
+
+ /*
+ * So we know it's in the general min/max, the question is whether it
+ * falls in one of the ranges or gaps. We'll use a binary search on
+ * the ranges.
+ *
+ * it's in the general range, but is it actually covered by any
+ * of the ranges? Repeat the check for each range.
+ *
+ * XXX We simply walk the ranges sequentially, but maybe we could
+ * further leverage the ordering and non-overlap and use bsearch to
+ * speed this up a bit.
+ */
+ start = 0; /* first range */
+ end = ranges->nranges - 1; /* last range */
+ while (true)
+ {
+ int midpoint = (start + end) / 2;
+
+ /* this means we ran out of ranges in the last step */
+ if (start > end)
+ return false;
+
+ /* copy the min/max values from the ranges */
+ minvalue = ranges->values[2 * midpoint];
+ maxvalue = ranges->values[2 * midpoint + 1];
+
+ /*
+ * Is the value smaller than the minval? If yes, we'll recurse
+ * to the left side of range array.
+ */
+ compar = FunctionCall2Coll(cmpLessFn, colloid, newval, minvalue);
+
+ /* smaller than the smallest value in this range */
+ if (DatumGetBool(compar))
+ {
+ end = (midpoint - 1);
+ continue;
+ }
+
+ /*
+ * Is the value greater than the minval? If yes, we'll recurse
+ * to the right side of range array.
+ */
+ compar = FunctionCall2Coll(cmpGreaterFn, colloid, newval, maxvalue);
+
+ /* larger than the largest value in this range */
+ if (DatumGetBool(compar))
+ {
+ start = (midpoint + 1);
+ continue;
+ }
+
+ /* hey, we found a matching range */
+ return true;
+ }
+
+ return false;
+}
+
+
/*
* range_contains_value
* See if the new value is already contained in the range list.
@@ -552,8 +737,6 @@ range_contains_value(BrinDesc *bdesc, Oid colloid,
Ranges *ranges, Datum newval)
{
int i;
- FmgrInfo *cmpLessFn;
- FmgrInfo *cmpGreaterFn;
FmgrInfo *cmpEqualFn;
Oid typid = attr->atttypid;
@@ -562,77 +745,8 @@ range_contains_value(BrinDesc *bdesc, Oid colloid,
* range, and only when there's still a chance of getting a match we
* inspect the individual ranges.
*/
- if (ranges->nranges > 0)
- {
- Datum compar;
- bool match = true;
-
- Datum minvalue = ranges->values[0];
- Datum maxvalue = ranges->values[2*ranges->nranges - 1];
-
- /*
- * Otherwise, need to compare the new value with boundaries of all
- * the ranges. First check if it's less than the absolute minimum,
- * which is the first value in the array.
- */
- cmpLessFn = minmax_multi_get_strategy_procinfo(bdesc, attno, typid,
- BTLessStrategyNumber);
- compar = FunctionCall2Coll(cmpLessFn, colloid, newval, minvalue);
-
- /* smaller than the smallest value in the range list */
- if (DatumGetBool(compar))
- match = false;
-
- /*
- * And now compare it to the existing maximum (last value in the
- * data array). But only if we haven't already ruled out a possible
- * match in the minvalue check.
- */
- if (match)
- {
- cmpGreaterFn = minmax_multi_get_strategy_procinfo(bdesc, attno, typid,
- BTGreaterStrategyNumber);
- compar = FunctionCall2Coll(cmpGreaterFn, colloid, newval, maxvalue);
-
- if (DatumGetBool(compar))
- match = false;
- }
-
- /*
- * So it's in the general range, but is it actually covered by any
- * of the ranges? Repeat the check for each range.
- *
- * XXX We simply walk the ranges sequentially, but maybe we could
- * further leverage the ordering and non-overlap and use bsearch to
- * speed this up a bit.
- */
- for (i = 0; i < ranges->nranges && match; i++)
- {
- /* copy the min/max values from the ranges */
- minvalue = ranges->values[2*i];
- maxvalue = ranges->values[2*i+1];
-
- /*
- * Otherwise, need to compare the new value with boundaries of all
- * the ranges. First check if it's less than the absolute minimum,
- * which is the first value in the array.
- */
- compar = FunctionCall2Coll(cmpLessFn, colloid, newval, minvalue);
-
- /* smaller than the smallest value in this range */
- if (DatumGetBool(compar))
- continue;
-
- compar = FunctionCall2Coll(cmpGreaterFn, colloid, newval, maxvalue);
-
- /* larger than the largest value in this range */
- if (DatumGetBool(compar))
- continue;
-
- /* hey, we found a matching row */
- return true;
- }
- }
+ if (has_matching_range(bdesc, colloid, ranges, newval, attno, typid))
+ return true;
cmpEqualFn = minmax_multi_get_strategy_procinfo(bdesc, attno, typid,
BTEqualStrategyNumber);
@@ -640,92 +754,42 @@ range_contains_value(BrinDesc *bdesc, Oid colloid,
/*
* We're done with the ranges, now let's inspect the exact values.
*
- * XXX Again, we do sequentially search the values - consider leveraging
- * the ordering of values to improve performance.
+ * XXX We do sequential search for small number of values, and bsearch
+ * once we have more than 16 values.
+ *
+ * XXX We only inspect the sorted part - that's OK. For building it may
+ * produce false negatives, but only after we already added some values
+ * to the unsorted part, so we've modified the value. And when querying
+ * the index, there should be no unsorted values.
*/
- for (i = 2*ranges->nranges; i < 2*ranges->nranges + ranges->nvalues; i++)
+ if (ranges->nsorted >= 16)
{
- Datum compar;
+ compare_context cxt;
- compar = FunctionCall2Coll(cmpEqualFn, colloid, newval, ranges->values[i]);
+ cxt.colloid = ranges->colloid;
+ cxt.cmpFn = ranges->cmp;
- /* found an exact match */
- if (DatumGetBool(compar))
+ if (bsearch_arg(&newval, &ranges->values[2*ranges->nranges],
+ ranges->nsorted, sizeof(Datum),
+ compare_values, (void *) &cxt) != NULL)
return true;
}
-
- /* the value is not covered by this BRIN tuple */
- return false;
-}
-
-/*
- * insert_value
- * Adds a new value into the single-point part, while maintaining ordering.
- *
- * The function inserts the new value to the right place in the single-point
- * part of the range. It assumes there's enough free space, and then does
- * essentially an insert-sort.
- *
- * XXX Assumes the 'values' array has space for (nvalues+1) entries, and that
- * only the first nvalues are used.
- */
-static void
-insert_value(FmgrInfo *cmp, Oid colloid, Datum *values, int nvalues,
- Datum newvalue)
-{
- int i;
- Datum lt;
-
- /* If there are no values yet, store the new one and we're done. */
- if (!nvalues)
+ else
{
- values[0] = newvalue;
- return;
- }
-
- /*
- * A common case is that the new value is entirely out of the existing
- * range, i.e. it's either smaller or larger than all previous values.
- * So we check and handle this case first - first we check the larger
- * case, because in that case we can just append the value to the end
- * of the array and we're done.
- */
+ for (i = 2*ranges->nranges; i < 2*ranges->nranges + ranges->nsorted; i++)
+ {
+ Datum compar;
- /* Is it greater than all existing values in the array? */
- lt = FunctionCall2Coll(cmp, colloid, values[nvalues-1], newvalue);
- if (DatumGetBool(lt))
- {
- /* just copy it in-place and we're done */
- values[nvalues] = newvalue;
- return;
- }
+ compar = FunctionCall2Coll(cmpEqualFn, colloid, newval, ranges->values[i]);
- /*
- * OK, I lied a bit - we won't check the smaller case explicitly, but
- * we'll just compare the value to all existing values in the array.
- * But we happen to start with the smallest value, so we're actually
- * doing the check anyway.
- *
- * XXX We do walk the values sequentially. Perhaps we could/should be
- * smarter and do some sort of bisection, to improve performance?
- */
- for (i = 0; i < nvalues; i++)
- {
- lt = FunctionCall2Coll(cmp, colloid, newvalue, values[i]);
- if (DatumGetBool(lt))
- {
- /*
- * Move values to make space for the new entry, which should go
- * to index 'i'. Entries 0 ... (i-1) should stay where they are.
- */
- memmove(&values[i+1], &values[i], (nvalues-i) * sizeof(Datum));
- values[i] = newvalue;
- return;
+ /* found an exact match */
+ if (DatumGetBool(compar))
+ return true;
}
}
- /* We should never really get here. */
- Assert(false);
+ /* the value is not covered by this BRIN tuple */
+ return false;
}
#ifdef USE_ASSERT_CHECKING
@@ -754,11 +818,12 @@ static void
AssertCheckRanges(Ranges *ranges, FmgrInfo *cmpFn, Oid colloid)
{
#ifdef USE_ASSERT_CHECKING
- int i, j;
+ int i;
/* some basic sanity checks */
Assert(ranges->nranges >= 0);
- Assert(ranges->nvalues >= 0);
+ Assert(ranges->nsorted >= 0);
+ Assert(ranges->nvalues >= ranges->nsorted);
Assert(ranges->maxvalues >= 2 * ranges->nranges + ranges->nvalues);
Assert(ranges->typid != InvalidOid);
@@ -770,32 +835,111 @@ AssertCheckRanges(Ranges *ranges, FmgrInfo *cmpFn, Oid colloid)
*/
AssertArrayOrder(cmpFn, colloid, ranges->values, 2*ranges->nranges);
- /* finally check that none of the values are not covered by ranges */
+ /* then the single-point ranges (with nvalues boundar values ) */
+ AssertArrayOrder(cmpFn, colloid, &ranges->values[2*ranges->nranges],
+ ranges->nsorted);
+
+ /*
+ * Check that none of the values are not covered by ranges (both
+ * sorted and unsorted)
+ */
for (i = 0; i < ranges->nvalues; i++)
{
+ Datum compar;
+ int start,
+ end;
+ Datum minvalue,
+ maxvalue;
+
Datum value = ranges->values[2 * ranges->nranges + i];
- for (j = 0; j < ranges->nranges; j++)
+ if (ranges->nranges == 0)
+ break;
+
+ minvalue = ranges->values[0];
+ maxvalue = ranges->values[2*ranges->nranges - 1];
+
+ /*
+ * Is the value smaller than the minval? If yes, we'll recurse
+ * to the left side of range array.
+ */
+ compar = FunctionCall2Coll(cmpFn, colloid, value, minvalue);
+
+ /* smaller than the smallest value in the first range */
+ if (DatumGetBool(compar))
+ continue;
+
+ /*
+ * Is the value greater than the minval? If yes, we'll recurse
+ * to the right side of range array.
+ */
+ compar = FunctionCall2Coll(cmpFn, colloid, maxvalue, value);
+
+ /* larger than the largest value in the last range */
+ if (DatumGetBool(compar))
+ continue;
+
+ start = 0; /* first range */
+ end = ranges->nranges - 1; /* last range */
+ while (true)
{
- Datum r;
+ int midpoint = (start + end) / 2;
+
+ /* this means we ran out of ranges in the last step */
+ if (start > end)
+ break;
+
+ /* copy the min/max values from the ranges */
+ minvalue = ranges->values[2 * midpoint];
+ maxvalue = ranges->values[2 * midpoint + 1];
- Datum minval = ranges->values[2 * j];
- Datum maxval = ranges->values[2 * j + 1];
+ /*
+ * Is the value smaller than the minval? If yes, we'll recurse
+ * to the left side of range array.
+ */
+ compar = FunctionCall2Coll(cmpFn, colloid, value, minvalue);
- /* if value is smaller than range minimum, that's OK */
- r = FunctionCall2Coll(cmpFn, colloid, value, minval);
- if (DatumGetBool(r))
+ /* smaller than the smallest value in this range */
+ if (DatumGetBool(compar))
+ {
+ end = (midpoint - 1);
continue;
+ }
+
+ /*
+ * Is the value greater than the minval? If yes, we'll recurse
+ * to the right side of range array.
+ */
+ compar = FunctionCall2Coll(cmpFn, colloid, maxvalue, value);
- /* if value is greater than range maximum, that's OK */
- r = FunctionCall2Coll(cmpFn, colloid, maxval, value);
- if (DatumGetBool(r))
+ /* larger than the largest value in this range */
+ if (DatumGetBool(compar))
+ {
+ start = (midpoint + 1);
continue;
+ }
- /* value is between [min,max], which is wrong */
+ /* hey, we found a matching range */
Assert(false);
}
}
+
+ /* and values in the unsorted part must not be in sorted part */
+ for (i = ranges->nsorted; i < ranges->nvalues; i++)
+ {
+ compare_context cxt;
+ Datum value = ranges->values[2 * ranges->nranges + i];
+
+ if (ranges->nsorted == 0)
+ break;
+
+ cxt.colloid = ranges->colloid;
+ cxt.cmpFn = ranges->cmp;
+
+ Assert(bsearch_arg(&value, &ranges->values[2*ranges->nranges],
+ ranges->nsorted, sizeof(Datum),
+ compare_values, (void *) &cxt) == NULL);
+ }
#endif
}
@@ -1106,8 +1250,7 @@ build_distances(FmgrInfo *distanceFn, Oid colloid,
*/
static CombineRange *
build_combine_ranges(FmgrInfo *cmp, Oid colloid, Ranges *ranges,
- bool addvalue, Datum newvalue, int *nranges,
- bool deduplicate)
+ int *nranges)
{
int ncranges;
CombineRange *cranges;
@@ -1115,28 +1258,15 @@ build_combine_ranges(FmgrInfo *cmp, Oid colloid, Ranges *ranges,
/* now do the actual merge sort */
ncranges = ranges->nranges + ranges->nvalues;
- /* should we add an extra value? */
- if (addvalue)
- ncranges += 1;
-
cranges = (CombineRange *) palloc0(ncranges * sizeof(CombineRange));
- /* put the new value at the beginning */
- if (addvalue)
- {
- cranges[0].minval = newvalue;
- cranges[0].maxval = newvalue;
- cranges[0].collapsed = true;
-
- /* then the regular and collapsed ranges */
- fill_combine_ranges(&cranges[1], ncranges-1, ranges);
- }
- else
- fill_combine_ranges(cranges, ncranges, ranges);
+ /* fll the combine ranges */
+ fill_combine_ranges(cranges, ncranges, ranges);
/* and sort the ranges */
- ncranges = sort_combine_ranges(cmp, colloid, cranges, ncranges,
- deduplicate);
+ ncranges = sort_combine_ranges(cmp, colloid,
+ cranges, ncranges,
+ true); /* deduplicate */
/* remember how many cranges we built */
*nranges = ncranges;
@@ -1321,19 +1451,28 @@ store_combine_ranges(Ranges *ranges, CombineRange *cranges, int ncranges)
}
}
+ /* all the values are sorted */
+ ranges->nsorted = ranges->nvalues;
+
Assert(count_values(cranges, ncranges) == 2*ranges->nranges + ranges->nvalues);
Assert(2*ranges->nranges + ranges->nvalues <= ranges->maxvalues);
}
+
+
/*
- * range_add_value
- * Add the new value to the multi-minmax range.
+ * Consider freeing space in the ranges.
+ *
+ * Returns true if the value was actually modified.
*/
static bool
-range_add_value(BrinDesc *bdesc, Oid colloid,
- AttrNumber attno, Form_pg_attribute attr,
- Ranges *ranges, Datum newval)
+ensure_free_space_in_buffer(BrinDesc *bdesc, Oid colloid,
+ AttrNumber attno, Form_pg_attribute attr,
+ Ranges *range)
{
+ MemoryContext ctx;
+ MemoryContext oldctx;
+
FmgrInfo *cmpFn,
*distanceFn;
@@ -1342,109 +1481,44 @@ range_add_value(BrinDesc *bdesc, Oid colloid,
int ncranges;
DistanceValue *distances;
- MemoryContext ctx;
- MemoryContext oldctx;
-
- /* we'll certainly need the comparator, so just look it up now */
- cmpFn = minmax_multi_get_strategy_procinfo(bdesc, attno, attr->atttypid,
- BTLessStrategyNumber);
-
- /* comprehensive checks of the input ranges */
- AssertCheckRanges(ranges, cmpFn, colloid);
-
- Assert((ranges->nranges >= 0) && (ranges->nvalues >= 0) && (ranges->maxvalues >= 0));
-
/*
- * When batch-building, there should be no ranges. So either the
- * number of ranges is 0 or we're not in batching mode.
+ * If there is free space in the buffer, we're done without having
+ * to modify anything.
*/
- Assert(!ranges->batch_mode || (ranges->nranges == 0));
-
- /* When batch-building, just add it and we're done. */
- if (ranges->batch_mode)
- {
- /* there has to be free space, if we've sized the struct */
- Assert(ranges->nvalues < ranges->maxvalues);
-
- /* Make a copy of the value, if needed. */
- ranges->values[ranges->nvalues++]
- = datumCopy(newval, attr->attbyval, attr->attlen);;
-
- return true;
- }
-
- /*
- * Bail out if the value already is covered by the range.
- *
- * We could also add values until we hit values_per_range, and then
- * do the deduplication in a batch, hoping for better efficiency. But
- * that would mean we actually modify the range every time, which means
- * having to serialize the value, which does palloc, walks the values,
- * copies them, etc. Not exactly cheap.
- *
- * So instead we do the check, which should be fairly cheap - assuming
- * the comparator function is not very expensive.
- *
- * This also implies means the values array can't contain duplicities.
- */
- if (range_contains_value(bdesc, colloid, attno, attr, ranges, newval))
+ if (2*range->nranges + range->nvalues < range->maxvalues)
return false;
- /* Make a copy of the value, if needed. */
- newval = datumCopy(newval, attr->attbyval, attr->attlen);
-
- /*
- * If there's space in the values array, copy it in and we're done.
- *
- * We do want to keep the values sorted (to speed up searches), so we
- * do a simple insertion sort. We could do something more elaborate,
- * e.g. by sorting the values only now and then, but for small counts
- * (e.g. when maxvalues is 64) this should be fine.
- */
- if (2*ranges->nranges + ranges->nvalues < ranges->maxvalues)
- {
- Datum *values;
-
- /* beginning of the 'single value' part (for convenience) */
- values = &ranges->values[2*ranges->nranges];
-
- insert_value(cmpFn, colloid, values, ranges->nvalues, newval);
-
- ranges->nvalues++;
-
- /*
- * Check we haven't broken the ordering of boundary values (checks
- * both parts, but that doesn't hurt).
- */
- AssertCheckRanges(ranges, cmpFn, colloid);
+ /* we'll certainly need the comparator, so just look it up now */
+ cmpFn = minmax_multi_get_strategy_procinfo(bdesc, attno, attr->atttypid,
+ BTLessStrategyNumber);
- /* Also check the range contains the value we just added. */
- // FIXME Assert(ranges, cmpFn, colloid);
+ /* Try deduplicating values in the unsorted part */
+ range_deduplicate_values(range);
- /* yep, we've modified the range */
+ /* did we reduce enough free space by just the deduplication? */
+ if (2*range->nranges + range->nvalues <= range->maxvalues * MINMAX_BUFFER_LOAD_FACTOR)
return true;
- }
/*
- * Damn - the new value is not in the range yet, but we don't have space
- * to just insert it. So we need to combine some of the existing ranges,
- * to reduce the number of values we need to store (joining two intervals
- * reduces the number of boundaries to store by 2).
+ * we need to combine some of the existing ranges, to reduce the number
+ * of values we need to store (joining intervals reduces the number of
+ * boundary values).
*
- * To do that we first construct an array of CombineRange items - each
- * combine range tracks if it's a regular range or collapsed range, where
- * "collapsed" means "single point."
+ * We first construct an array of CombineRange items - each combine range
+ * tracks if it's a regular range or a collapsed range, where "collapsed"
+ * means "single point." This makes the processing easier, as it allows
+ * handling ranges and points the same way.
*
- * Existing ranges (we have ranges->nranges of them) map to combine ranges
- * directly, while single points (ranges->nvalues of them) have to be
- * expanded. We neet the combine ranges to be sorted, and we do that by
- * performing a merge sort of ranges, values and new value.
+ * Then we sort the combine ranges - this is necessary, because although
+ * ranges and points were sorted on their own, the new array is not. We
+ * do that by performing a merge sort of the two parts.
*
* The distanceFn calls (which may internally call e.g. numeric_le) may
- * allocate quite a bit of memory, and we must not leak it. Otherwise
- * we'd have problems e.g. when building indexes. So we create a local
- * memory context and make sure we free the memory before leaving this
- * function (not after every call).
+ * allocate quite a bit of memory, and we must not leak it (we might have
+ * to do this repeatedly, even for a single BRIN page range). Otherwise
+ * we'd have problems e.g. when building new indexes. So we use a memory
+ * context and make sure we free the memory at the end (so if we call
+ * the distance function many times, it might be an issue, but meh).
*/
ctx = AllocSetContextCreate(CurrentMemoryContext,
"minmax-multi context",
@@ -1453,9 +1527,7 @@ range_add_value(BrinDesc *bdesc, Oid colloid,
oldctx = MemoryContextSwitchTo(ctx);
/* OK build the combine ranges */
- cranges = build_combine_ranges(cmpFn, colloid, ranges,
- true, newval, &ncranges,
- false);
+ cranges = build_combine_ranges(cmpFn, colloid, range, &ncranges);
/* and we'll also need the 'distance' procedure */
distanceFn = minmax_multi_get_procinfo(bdesc, attno, PROCNUM_DISTANCE);
@@ -1469,21 +1541,104 @@ range_add_value(BrinDesc *bdesc, Oid colloid,
* use too low or high value.
*/
ncranges = reduce_combine_ranges(cranges, ncranges, distances,
- ranges->maxvalues * MINMAX_LOAD_FACTOR,
+ range->maxvalues * MINMAX_BUFFER_LOAD_FACTOR,
cmpFn, colloid);
- Assert(count_values(cranges, ncranges) <= ranges->maxvalues * MINMAX_LOAD_FACTOR);
+ Assert(count_values(cranges, ncranges) <= range->maxvalues * MINMAX_BUFFER_LOAD_FACTOR);
/* decompose the combine ranges into regular ranges and single values */
- store_combine_ranges(ranges, cranges, ncranges);
+ store_combine_ranges(range, cranges, ncranges);
MemoryContextSwitchTo(oldctx);
MemoryContextDelete(ctx);
/* Did we break the ranges somehow? */
+ AssertCheckRanges(range, cmpFn, colloid);
+
+ return true;
+}
+
+/*
+ * range_add_value
+ * Add the new value to the multi-minmax range.
+ */
+static bool
+range_add_value(BrinDesc *bdesc, Oid colloid,
+ AttrNumber attno, Form_pg_attribute attr,
+ Ranges *ranges, Datum newval)
+{
+ FmgrInfo *cmpFn;
+ bool modified = false;
+
+ /* we'll certainly need the comparator, so just look it up now */
+ cmpFn = minmax_multi_get_strategy_procinfo(bdesc, attno, attr->atttypid,
+ BTLessStrategyNumber);
+
+ /* comprehensive checks of the input ranges */
AssertCheckRanges(ranges, cmpFn, colloid);
+
+ /*
+ * Make sure there's enough free space in the buffer. We only trigger
+ * this when the buffer is full, which means it had to be modified as
+ * we size it to be larger than what is stored on disk.
+ *
+ * XXX This needs to happen before we check if the value is contained
+ * in the range, because the value might be in the unsorted part, and
+ * we don't check that in range_contains_value. The deduplication would
+ * then move it to the sorted part, and we'd add the value too, which
+ * violates the rule that we never have duplicates with the ranges
+ * or sorted values.
+ *
+ * XXX At the moment this only does the deduplication.
+ *
+ * XXX We might also deduplicate and recheck if the value is contained,
+ * but that seems like an overkill. We'd need to deduplicate anyway,
+ * so why not do it now.
+ */
+ modified = ensure_free_space_in_buffer(bdesc, colloid,
+ attno, attr, ranges);
+
+ /*
+ * Bail out if the value already is covered by the range.
+ *
+ * We could also add values until we hit values_per_range, and then
+ * do the deduplication in a batch, hoping for better efficiency. But
+ * that would mean we actually modify the range every time, which means
+ * having to serialize the value, which does palloc, walks the values,
+ * copies them, etc. Not exactly cheap.
+ *
+ * So instead we do the check, which should be fairly cheap - assuming
+ * the comparator function is not very expensive.
+ *
+ * This also implies means the values array can't contain duplicities.
+ */
+ if (range_contains_value(bdesc, colloid, attno, attr, ranges, newval))
+ return modified;
+
+ /* Make a copy of the value, if needed. */
+ newval = datumCopy(newval, attr->attbyval, attr->attlen);
+
+ /*
+ * If there's space in the values array, copy it in and we're done.
+ *
+ * We do want to keep the values sorted (to speed up searches), so we
+ * do a simple insertion sort. We could do something more elaborate,
+ * e.g. by sorting the values only now and then, but for small counts
+ * (e.g. when maxvalues is 64) this should be fine.
+ */
+ ranges->values[2*ranges->nranges + ranges->nvalues] = newval;
+ ranges->nvalues++;
+
+ /*
+ * Check we haven't broken the ordering of boundary values (checks
+ * both parts, but that doesn't hurt).
+ */
+ AssertCheckRanges(ranges, cmpFn, colloid);
+
+ /* Also check the range contains the value we just added. */
// FIXME Assert(ranges, cmpFn, colloid);
+ /* yep, we've modified the range */
return true;
}
@@ -1506,12 +1661,6 @@ compactify_ranges(BrinDesc *bdesc, Ranges *ranges, int max_values)
MemoryContext ctx;
MemoryContext oldctx;
- /*
- * This should only be used in batch mode, and there should be no
- * ranges, just individual values.
- */
- Assert((ranges->batch_mode) && (ranges->nranges == 0));
-
/* we'll certainly need the comparator, so just look it up now */
cmpFn = minmax_multi_get_strategy_procinfo(bdesc, ranges->attno, ranges->typid,
BTLessStrategyNumber);
@@ -1534,8 +1683,7 @@ compactify_ranges(BrinDesc *bdesc, Ranges *ranges, int max_values)
/* OK build the combine ranges */
cranges = build_combine_ranges(cmpFn, ranges->colloid, ranges,
- false, (Datum) 0, &ncranges,
- true); /* deduplicate */
+ &ncranges); /* deduplicate */
if (ncranges > 1)
{
@@ -1548,7 +1696,7 @@ compactify_ranges(BrinDesc *bdesc, Ranges *ranges, int max_values)
* don't expect more tuples to be inserted soon.
*/
ncranges = reduce_combine_ranges(cranges, ncranges, distances,
- max_values, cmpFn, ranges->colloid);
+ max_values, cmpFn, ranges->colloid);
Assert(count_values(cranges, ncranges) <= max_values);
}
@@ -2052,8 +2200,7 @@ brin_minmax_multi_serialize(BrinDesc *bdesc, Datum src, Datum *dst)
* In batch mode, we need to compress the accumulated values to the
* actually requested number of values/ranges.
*/
- if (ranges->batch_mode)
- compactify_ranges(bdesc, ranges, ranges->target_maxvalues);
+ compactify_ranges(bdesc, ranges, ranges->target_maxvalues);
s = range_serialize(ranges);
dst[0] = PointerGetDatum(s);
@@ -2114,15 +2261,39 @@ brin_minmax_multi_add_value(PG_FUNCTION_ARGS)
{
MemoryContext oldctx;
+ int target_maxvalues;
+ int maxvalues;
BlockNumber pagesPerRange = BrinGetPagesPerRange(bdesc->bd_index);
+ /* what was specified as a reloption? */
+ target_maxvalues = brin_minmax_multi_get_values(bdesc, opts);
+
+ /*
+ * Determine the insert buffer size - we use 10x the target, capped
+ * to the maximum number of values in the heap range. This is more
+ * than enough, considering the actual number of rows per page is
+ * likely much lower, but meh.
+ */
+ maxvalues = Min(target_maxvalues * MINMAX_BUFFER_FACTOR,
+ MaxHeapTuplesPerPage * pagesPerRange);
+
+ /* but always at least the original value */
+ maxvalues = Max(maxvalues, target_maxvalues);
+
+ /* always cap by MIN/MAX */
+ maxvalues = Max(maxvalues, MINMAX_BUFFER_MIN);
+ maxvalues = Min(maxvalues, MINMAX_BUFFER_MAX);
+
oldctx = MemoryContextSwitchTo(column->bv_context);
- ranges = minmax_multi_init(MaxHeapTuplesPerPage * pagesPerRange);
+ ranges = minmax_multi_init(maxvalues);
ranges->attno = attno;
ranges->colloid = colloid;
ranges->typid = attr->atttypid;
- ranges->batch_mode = true;
- ranges->target_maxvalues = brin_minmax_multi_get_values(bdesc, opts);
+ ranges->target_maxvalues = target_maxvalues;
+
+ /* we'll certainly need the comparator, so just look it up now */
+ ranges->cmp = minmax_multi_get_strategy_procinfo(bdesc, attno, attr->atttypid,
+ BTLessStrategyNumber);
MemoryContextSwitchTo(oldctx);
@@ -2136,10 +2307,38 @@ brin_minmax_multi_add_value(PG_FUNCTION_ARGS)
{
MemoryContext oldctx;
+ int maxvalues;
+ BlockNumber pagesPerRange = BrinGetPagesPerRange(bdesc->bd_index);
+
oldctx = MemoryContextSwitchTo(column->bv_context);
serialized = (SerializedRanges *) PG_DETOAST_DATUM(column->bv_values[0]);
- ranges = range_deserialize(serialized);
+
+ /*
+ * Determine the insert buffer size - we use 10x the target, capped
+ * to the maximum number of values in the heap range. This is more
+ * than enough, considering the actual number of rows per page is
+ * likely much lower, but meh.
+ */
+ maxvalues = Min(serialized->maxvalues * MINMAX_BUFFER_FACTOR,
+ MaxHeapTuplesPerPage * pagesPerRange);
+
+ /* but always at least the original value */
+ maxvalues = Max(maxvalues, serialized->maxvalues);
+
+ /* always cap by MIN/MAX */
+ maxvalues = Max(maxvalues, MINMAX_BUFFER_MIN);
+ maxvalues = Min(maxvalues, MINMAX_BUFFER_MAX);
+
+ ranges = range_deserialize(maxvalues, serialized);
+
+ ranges->attno = attno;
+ ranges->colloid = colloid;
+ ranges->typid = attr->atttypid;
+
+ /* we'll certainly need the comparator, so just look it up now */
+ ranges->cmp = minmax_multi_get_strategy_procinfo(bdesc, attno, attr->atttypid,
+ BTLessStrategyNumber);
column->bv_mem_value = PointerGetDatum(ranges);
column->bv_serialize = brin_minmax_multi_serialize;
@@ -2184,7 +2383,7 @@ brin_minmax_multi_consistent(PG_FUNCTION_ARGS)
attno = column->bv_attno;
serialized = (SerializedRanges *) PG_DETOAST_DATUM(column->bv_values[0]);
- ranges = range_deserialize(serialized);
+ ranges = range_deserialize(serialized->maxvalues, serialized);
/* inspect the ranges, and for each one evaluate the scan keys */
for (rangeno = 0; rangeno < ranges->nranges; rangeno++)
@@ -2371,8 +2570,8 @@ brin_minmax_multi_union(PG_FUNCTION_ARGS)
serialized_a = (SerializedRanges *) PG_DETOAST_DATUM(col_a->bv_values[0]);
serialized_b = (SerializedRanges *) PG_DETOAST_DATUM(col_b->bv_values[0]);
- ranges_a = range_deserialize(serialized_a);
- ranges_b = range_deserialize(serialized_b);
+ ranges_a = range_deserialize(serialized_a->maxvalues, serialized_a);
+ ranges_b = range_deserialize(serialized_b->maxvalues, serialized_b);
/* make sure neither of the ranges is NULL */
Assert(ranges_a && ranges_b);
@@ -2408,7 +2607,7 @@ brin_minmax_multi_union(PG_FUNCTION_ARGS)
cmpFn = minmax_multi_get_strategy_procinfo(bdesc, attno, attr->atttypid,
BTLessStrategyNumber);
- /* sort the combine ranges (don't deduplicate) */
+ /* sort the combine ranges (no need to deduplicate) */
sort_combine_ranges(cmpFn, colloid, cranges, ncranges, false);
/*
@@ -2637,7 +2836,7 @@ brin_minmax_multi_summary_out(PG_FUNCTION_ARGS)
fmgr_info(outfunc, &fmgrinfo);
/* deserialize the range info easy-to-process pieces */
- ranges_deserialized = range_deserialize(ranges);
+ ranges_deserialized = range_deserialize(ranges->maxvalues, ranges);
appendStringInfo(&str, "nranges: %u nvalues: %u maxvalues: %u",
ranges_deserialized->nranges,
--
2.26.2
--------------0E72B707603BED22B4040825
Content-Type: text/x-patch; charset=UTF-8;
name="0006-Batch-mode-when-building-new-BRIN-multi-min-20210211.patch"
Content-Transfer-Encoding: 7bit
Content-Disposition: attachment;
filename*0="0006-Batch-mode-when-building-new-BRIN-multi-min-20210211.pa";
filename*1="tch"
^ permalink raw reply [nested|flat] 19+ messages in thread
* Re: daitch_mokotoff module
@ 2022-01-03 19:16 Andres Freund <[email protected]>
2022-01-04 13:49 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
0 siblings, 1 reply; 19+ messages in thread
From: Andres Freund @ 2022-01-03 19:16 UTC (permalink / raw)
To: Tom Lane <[email protected]>; +Cc: Thomas Munro <[email protected]>; Dag Lem <[email protected]>; PostgreSQL Hackers <[email protected]>
Hi,
On 2022-01-02 21:41:53 -0500, Tom Lane wrote:
> ... so, that test case is guaranteed to fail in non-UTF8 encodings,
> I suppose? I wonder what the LANG environment is in that cfbot
> instance.
LANG="en_US.UTF-8"
But it looks to me like the problem is in the commit cfbot creates, rather
than the test run itself:
https://github.com/postgresql-cfbot/postgresql/commit/d5b4ec87cfd65dc08d26e1b789bd254405c90a66#diff-...
Greetings,
Andres Freund
^ permalink raw reply [nested|flat] 19+ messages in thread
* Re: daitch_mokotoff module
2022-01-03 19:16 Re: daitch_mokotoff module Andres Freund <[email protected]>
@ 2022-01-04 13:49 ` Dag Lem <[email protected]>
2022-01-04 20:18 ` Re: daitch_mokotoff module Thomas Munro <[email protected]>
0 siblings, 1 reply; 19+ messages in thread
From: Dag Lem @ 2022-01-04 13:49 UTC (permalink / raw)
To: Andres Freund <[email protected]>; +Cc: Tom Lane <[email protected]>; Thomas Munro <[email protected]>; PostgreSQL Hackers <[email protected]>
Andres Freund <[email protected]> writes:
> Hi,
>
> On 2022-01-02 21:41:53 -0500, Tom Lane wrote:
>> ... so, that test case is guaranteed to fail in non-UTF8 encodings,
>> I suppose? I wonder what the LANG environment is in that cfbot
>> instance.
>
> LANG="en_US.UTF-8"
>
> But it looks to me like the problem is in the commit cfbot creates, rather
> than the test run itself:
> https://github.com/postgresql-cfbot/postgresql/commit/d5b4ec87cfd65dc08d26e1b789bd254405c90a66#diff-...
>
> Greetings,
>
> Andres Freund
>
>
I have now separated out the UTF8-dependent tests, hopefully according
to the current best practice (based on src/test/modules/test_regex/ and
https://www.postgresql.org/docs/14/regress-variant.html).
However I guess this won't make any difference wrt. actually running the
tests, as long as there seems to be an encoding problem in the cfbot
pipeline.
Is there anything else I can do? Could perhaps fuzzystrmatch_utf8 simply
be commented out from the Makefile for the time being?
Best regards
Dag Lem
Attachments:
[text/x-patch] v5-daitch_mokotoff.patch (51.2K, ../../[email protected]/2-v5-daitch_mokotoff.patch)
download | inline diff:
diff --git a/contrib/fuzzystrmatch/Makefile b/contrib/fuzzystrmatch/Makefile
index 0704894f88..1d5bd84be8 100644
--- a/contrib/fuzzystrmatch/Makefile
+++ b/contrib/fuzzystrmatch/Makefile
@@ -3,14 +3,15 @@
MODULE_big = fuzzystrmatch
OBJS = \
$(WIN32RES) \
+ daitch_mokotoff.o \
dmetaphone.o \
fuzzystrmatch.o
EXTENSION = fuzzystrmatch
-DATA = fuzzystrmatch--1.1.sql fuzzystrmatch--1.0--1.1.sql
+DATA = fuzzystrmatch--1.2.sql fuzzystrmatch--1.1--1.2.sql fuzzystrmatch--1.0--1.1.sql
PGFILEDESC = "fuzzystrmatch - similarities and distance between strings"
-REGRESS = fuzzystrmatch
+REGRESS = fuzzystrmatch fuzzystrmatch_utf8
ifdef USE_PGXS
PG_CONFIG = pg_config
@@ -22,3 +23,8 @@ top_builddir = ../..
include $(top_builddir)/src/Makefile.global
include $(top_srcdir)/contrib/contrib-global.mk
endif
+
+daitch_mokotoff.o: daitch_mokotoff.h
+
+daitch_mokotoff.h: daitch_mokotoff_header.pl
+ perl $< > $@
diff --git a/contrib/fuzzystrmatch/daitch_mokotoff.c b/contrib/fuzzystrmatch/daitch_mokotoff.c
new file mode 100644
index 0000000000..1b7263c349
--- /dev/null
+++ b/contrib/fuzzystrmatch/daitch_mokotoff.c
@@ -0,0 +1,593 @@
+/*
+ * Daitch-Mokotoff Soundex
+ *
+ * Copyright (c) 2021 Finance Norway
+ * Author: Dag Lem <[email protected]>
+ *
+ * This implementation of the Daitch-Mokotoff Soundex System aims at high
+ * performance.
+ *
+ * - The processing of each phoneme is initiated by an O(1) table lookup.
+ * - For phonemes containing more than one character, a coding tree is traversed
+ * to process the complete phoneme.
+ * - The (alternate) soundex codes are produced digit by digit in-place in
+ * another tree structure.
+ *
+ * References:
+ *
+ * https://www.avotaynu.com/soundex.htm
+ * https://www.jewishgen.org/InfoFiles/Soundex.html
+ * https://familypedia.fandom.com/wiki/Daitch-Mokotoff_Soundex
+ * https://stevemorse.org/census/soundex.html (dmlat.php, dmsoundex.php)
+ * https://github.com/apache/commons-codec/ (dmrules.txt, DaitchMokotoffSoundex.java)
+ * https://metacpan.org/pod/Text::Phonetic (DaitchMokotoff.pm)
+ *
+ * A few notes on other implementations:
+ *
+ * - All other known implementations have the same unofficial rules for "UE",
+ * these are also adapted by this implementation (0, 1, NC).
+ * - The only other known implementation which is capable of generating all
+ * correct soundex codes in all cases is the JOS Soundex Calculator at
+ * https://www.jewishgen.org/jos/jossound.htm
+ * - "J" is considered (only) a vowel in dmlat.php
+ * - The official rules for "RS" are commented out in dmlat.php
+ * - Identical code digits for adjacent letters are not collapsed correctly in
+ * dmsoundex.php when double digit codes are involved. E.g. "BESST" yields
+ * 744300 instead of 743000 as for "BEST".
+ * - "J" is considered (only) a consonant in DaitchMokotoffSoundex.java
+ * - "Y" is not considered a vowel in DaitchMokotoffSoundex.java
+ *
+ * Permission to use, copy, modify, and distribute this software and its
+ * documentation for any purpose, without fee, and without a written agreement
+ * is hereby granted, provided that the above copyright notice and this
+ * paragraph and the following two paragraphs appear in all copies.
+ *
+ * IN NO EVENT SHALL THE AUTHOR OR DISTRIBUTORS BE LIABLE TO ANY PARTY FOR
+ * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING
+ * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS
+ * DOCUMENTATION, EVEN IF THE AUTHOR OR DISTRIBUTORS HAVE BEEN ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ * THE AUTHOR AND DISTRIBUTORS SPECIFICALLY DISCLAIM ANY WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
+ * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS
+ * ON AN "AS IS" BASIS, AND THE AUTHOR AND DISTRIBUTORS HAS NO OBLIGATIONS TO
+ * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
+*/
+
+#include "daitch_mokotoff.h"
+
+#include "postgres.h"
+#include "utils/builtins.h"
+#include "mb/pg_wchar.h"
+
+#include <ctype.h>
+#include <string.h>
+
+/* Internal C implementation */
+static char *_daitch_mokotoff(char *word, char *soundex, size_t n);
+
+
+PG_FUNCTION_INFO_V1(daitch_mokotoff);
+Datum
+daitch_mokotoff(PG_FUNCTION_ARGS)
+{
+ text *arg = PG_GETARG_TEXT_PP(0);
+ char *string,
+ *tmp_soundex;
+ text *soundex;
+
+ /*
+ * The maximum theoretical soundex size is several KB, however in practice
+ * anything but contrived synthetic inputs will yield a soundex size of
+ * less than 100 bytes. We thus allocate and free a temporary work buffer,
+ * and return only the actual soundex result.
+ */
+ string = pg_server_to_any(text_to_cstring(arg), VARSIZE_ANY_EXHDR(arg), PG_UTF8);
+ tmp_soundex = palloc(DM_MAX_SOUNDEX_CHARS);
+
+ if (!_daitch_mokotoff(string, tmp_soundex, DM_MAX_SOUNDEX_CHARS))
+ {
+ /* No encodable characters in input. */
+ pfree(tmp_soundex);
+ PG_RETURN_NULL();
+ }
+
+ soundex = cstring_to_text(pg_any_to_server(tmp_soundex, strlen(tmp_soundex), PG_UTF8));
+ pfree(tmp_soundex);
+
+ PG_RETURN_TEXT_P(soundex);
+}
+
+
+typedef dm_node dm_nodes[DM_MAX_NODES];
+typedef dm_node * dm_leaves[DM_MAX_LEAVES];
+
+
+/* Template for new node in soundex code tree. */
+static const dm_node start_node = {
+ .soundex_length = 0,
+ .soundex = "000000 ", /* Six digits + joining space */
+ .is_leaf = 0,
+ .last_update = 0,
+ .code_digit = '\0',
+ .prev_code_digits = {'\0', '\0'},
+ .next_code_digits = {'\0', '\0'},
+ .prev_code_index = 0,
+ .next_code_index = 0,
+ .next_nodes = {NULL}
+};
+
+/* Dummy soundex codes at end of input. */
+static dm_codes end_codes[2] =
+{
+ {
+ "X", "X", "X"
+ }
+};
+
+
+/* Initialize soundex code tree node for next code digit. */
+static void
+initialize_node(dm_node * node, int last_update)
+{
+ if (node->last_update < last_update)
+ {
+ node->prev_code_digits[0] = node->next_code_digits[0];
+ node->prev_code_digits[1] = node->next_code_digits[1];
+ node->next_code_digits[0] = '\0';
+ node->next_code_digits[1] = '\0';
+ node->prev_code_index = node->next_code_index;
+ node->next_code_index = 0;
+ node->is_leaf = 0;
+ node->last_update = last_update;
+ }
+}
+
+
+/* Update soundex code tree node with next code digit. */
+static void
+add_next_code_digit(dm_node * node, int code_index, char code_digit)
+{
+ /* OR in index 1 or 2. */
+ node->next_code_index |= code_index;
+
+ if (!node->next_code_digits[0])
+ {
+ node->next_code_digits[0] = code_digit;
+ }
+ else if (node->next_code_digits[0] != code_digit)
+ {
+ node->next_code_digits[1] = code_digit;
+ }
+}
+
+
+/* Mark soundex code tree node as leaf. */
+static void
+set_leaf(dm_leaves leaves_next, int *num_leaves_next, dm_node * node)
+{
+ if (!node->is_leaf)
+ {
+ node->is_leaf = 1;
+ leaves_next[(*num_leaves_next)++] = node;
+ }
+}
+
+
+/* Find next node corresponding to code digit, or create a new node. */
+static dm_node * find_or_create_node(dm_nodes nodes, int *num_nodes,
+ dm_node * node, char code_digit)
+{
+ dm_node **next_nodes;
+ dm_node *next_node;
+
+ for (next_nodes = node->next_nodes; (next_node = *next_nodes); next_nodes++)
+ {
+ if (next_node->code_digit == code_digit)
+ {
+ return next_node;
+ }
+ }
+
+ next_node = &nodes[(*num_nodes)++];
+ *next_nodes = next_node;
+
+ *next_node = start_node;
+ memcpy(next_node->soundex, node->soundex, sizeof(next_node->soundex));
+ next_node->soundex_length = node->soundex_length;
+ next_node->soundex[next_node->soundex_length++] = code_digit;
+ next_node->code_digit = code_digit;
+ next_node->next_code_index = node->prev_code_index;
+
+ return next_node;
+}
+
+
+/* Update node for next code digit(s). */
+static int
+update_node(dm_nodes nodes, dm_node * node, int *num_nodes,
+ dm_leaves leaves_next, int *num_leaves_next,
+ int letter_no, int prev_code_index, int next_code_index,
+ char *next_code_digits, int digit_no)
+{
+ int i;
+ char next_code_digit = next_code_digits[digit_no];
+ int num_dirty_nodes = 0;
+ dm_node *dirty_nodes[2];
+
+ initialize_node(node, letter_no);
+
+ if (node->soundex_length == DM_MAX_CODE_DIGITS)
+ {
+ /* Keep completed soundex code. */
+ set_leaf(leaves_next, num_leaves_next, node);
+ return 0;
+ }
+
+ if (node->prev_code_index && !(node->prev_code_index & prev_code_index))
+ {
+ /*
+ * If the sound (vowel / consonant) of this letter encoding doesn't
+ * correspond to the coding index of the previous letter, we skip this
+ * letter encoding. Note that currently, only "J" can be either a
+ * vowel or a consonant.
+ */
+ return 1;
+ }
+
+ if (next_code_digit == 'X' ||
+ (digit_no == 0 &&
+ (node->prev_code_digits[0] == next_code_digit ||
+ node->prev_code_digits[1] == next_code_digit)))
+ {
+ /* The code digit is the same as one of the previous (i.e. not added). */
+ dirty_nodes[num_dirty_nodes++] = node;
+ }
+
+ if (next_code_digit != 'X' &&
+ (digit_no > 0 ||
+ node->prev_code_digits[0] != next_code_digit ||
+ node->prev_code_digits[1]))
+ {
+ /* The code digit is different from one of the previous (i.e. added). */
+ node = find_or_create_node(nodes, num_nodes, node, next_code_digit);
+ initialize_node(node, letter_no);
+ dirty_nodes[num_dirty_nodes++] = node;
+ }
+
+ for (i = 0; i < num_dirty_nodes; i++)
+ {
+ /* Add code digit leading to the current node. */
+ add_next_code_digit(dirty_nodes[i], next_code_index, next_code_digit);
+
+ if (next_code_digits[++digit_no])
+ {
+ update_node(nodes, dirty_nodes[i], num_nodes,
+ leaves_next, num_leaves_next,
+ letter_no, prev_code_index, next_code_index,
+ next_code_digits, digit_no);
+ }
+ else
+ {
+ set_leaf(leaves_next, num_leaves_next, dirty_nodes[i]);
+ }
+ }
+
+ return 1;
+}
+
+
+/* Update soundex tree leaf nodes. Return 1 when all nodes are completed. */
+static int
+update_leaves(dm_nodes nodes, int *num_nodes,
+ dm_leaves leaves[2], int *ix_leaves, int *num_leaves,
+ int letter_no, dm_codes * codes, dm_codes * next_codes)
+{
+ int i,
+ j,
+ k,
+ code_index;
+ dm_code *code,
+ *next_code;
+ int num_leaves_next = 0;
+ int ix_leaves_next = (*ix_leaves + 1) & 1; /* Alternate ix: 0, 1 */
+ int finished = 1;
+
+ for (i = 0; i < *num_leaves; i++)
+ {
+ dm_node *node = leaves[*ix_leaves][i];
+
+ /* One or two alternate code sequences. */
+ for (j = 0; j < 2 && (code = codes[j]) && code[0][0]; j++)
+ {
+ /* Coding for previous letter - before vowel: 1, all other: 2 */
+ int prev_code_index = (code[0][0] > '1') + 1;
+
+ /* One or two alternate next code sequences. */
+ for (k = 0; k < 2 && (next_code = next_codes[k]) && next_code[0][0]; k++)
+ {
+ /* Determine which code to use. */
+ if (letter_no == 0)
+ {
+ /* This is the first letter. */
+ code_index = 0;
+ }
+ else if (next_code[0][0] <= '1')
+ {
+ /* The next letter is a vowel. */
+ code_index = 1;
+ }
+ else
+ {
+ /* All other cases. */
+ code_index = 2;
+ }
+
+ /* One or two sequential code digits. */
+ if (update_node(nodes, node, num_nodes,
+ leaves[ix_leaves_next], &num_leaves_next,
+ letter_no, prev_code_index, code_index,
+ code[code_index], 0))
+ {
+ finished = 0;
+ }
+ }
+ }
+ }
+
+ *ix_leaves = ix_leaves_next;
+ *num_leaves = num_leaves_next;
+
+ return finished;
+}
+
+
+/* Mapping from ISO8859-1 to ASCII */
+static const char tr_accents_iso8859_1[] =
+/*
+"ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖ×ØÙÚÛÜÝÞßàáâãäåæçèéêëìíîïðñòóôõö÷øùúûüýþÿ"
+*/
+"AAAAAAECEEEEIIIIDNOOOOO*OUUUUYDsaaaaaaeceeeeiiiidnooooo/ouuuuydy";
+
+static char
+unaccent_iso8859_1(unsigned char c)
+{
+ return c >= 192 ? tr_accents_iso8859_1[c - 192] : c;
+}
+
+
+/* Convert an UTF-8 character to ISO-8859-1.
+ * Unconvertable characters are returned as '?'.
+ * NB! Beware of the domain specific conversion of Ą, Ę, and Ţ/Ț.
+ */
+static char
+utf8_to_iso8859_1(char *str, int *ix)
+{
+ const char unknown = '?';
+ unsigned char c,
+ c2;
+ unsigned int code_point;
+
+ /* First byte. */
+ c = (unsigned char) str[(*ix)++];
+ if (c < 0x80)
+ {
+ /* ASCII code point. */
+ if (c >= '[' && c <= ']')
+ {
+ /* Codes reserved for Ą, Ę, and Ţ/Ț. */
+ return unknown;
+ }
+
+ return c;
+ }
+
+ /* Second byte. */
+ c2 = (unsigned char) str[(*ix)++];
+ if (!c2)
+ {
+ /* The UTF-8 character is cut short (invalid code point). */
+ (*ix)--;
+ return unknown;
+ }
+
+ if (c < 0xE0)
+ {
+ /* Two-byte character. */
+ code_point = ((c & 0x1F) << 6) | (c2 & 0x3F);
+ if (code_point < 0x100)
+ {
+ /* ISO-8859-1 code point. */
+ return code_point;
+ }
+ else if (code_point == 0x0104 || code_point == 0x0105)
+ {
+ /* Ą/ą */
+ return '[';
+ }
+ else if (code_point == 0x0118 || code_point == 0x0119)
+ {
+ /* Ę/ę */
+ return '\\';
+ }
+ else if (code_point == 0x0162 || code_point == 0x0163 ||
+ code_point == 0x021A || code_point == 0x021B)
+ {
+ /* Ţ/ţ or Ț/ț */
+ return ']';
+ }
+ else
+ {
+ return unknown;
+ }
+ }
+
+ /* Third byte. */
+ if (!str[(*ix)++])
+ {
+ /* The UTF-8 character is cut short (invalid code point). */
+ (*ix)--;
+ return unknown;
+ }
+
+ if (c < 0xF0)
+ {
+ /* Three-byte character. */
+ return unknown;
+ }
+
+ /* Fourth byte. */
+ if (!str[(*ix)++])
+ {
+ /* The UTF-8 character is cut short (invalid code point). */
+ (*ix)--;
+ }
+
+ return unknown;
+}
+
+
+/* Return next character, converted from UTF-8 to uppercase ASCII. */
+static char
+read_char(char *str, int *ix)
+{
+ return toupper(unaccent_iso8859_1(utf8_to_iso8859_1(str, ix)));
+}
+
+
+/* Convert input to ASCII, skipping any characters not in [A-\]]. */
+static void
+normalize_input(char *src, char *dst)
+{
+ int c;
+ int i = 0,
+ j = 0;
+
+ while ((c = read_char(src, &i)))
+ {
+ if (c >= 'A' && c <= ']')
+ {
+ dst[j++] = c;
+ }
+ }
+
+ dst[j] = '\0';
+}
+
+
+/* Return sound coding for "letter" (letter sequence) */
+static dm_codes *
+read_letter(char *str, int *ix)
+{
+ int c,
+ cmp;
+ int i = *ix,
+ j;
+ dm_letter *letters;
+ dm_codes *codes;
+
+ /* First letter in sequence. */
+ if (!(c = str[i++]))
+ {
+ return NULL;
+ }
+ letters = &letter_[c - 'A'];
+ codes = letters->codes;
+ *ix = i;
+
+ /* Any subsequent letters in sequence. */
+ while ((letters = letters->letters) && (c = str[i++]))
+ {
+ for (j = 0; (cmp = letters[j].letter); j++)
+ {
+ if (cmp == c)
+ {
+ /* Letter found. */
+ letters = &letters[j];
+ if (letters->codes)
+ {
+ /* Coding for letter sequence found. */
+ codes = letters->codes;
+ *ix = i;
+ }
+ break;
+ }
+ }
+ if (!cmp)
+ {
+ /* The sequence of letters has no coding. */
+ break;
+ }
+ }
+
+ return codes;
+}
+
+
+/* Generate all Daitch-Mokotoff soundex codes for word, separated by space. */
+static char *
+_daitch_mokotoff(char *word, char *soundex, size_t n)
+{
+ int i = 0,
+ j;
+ int letter_no = 0;
+ int ix_leaves = 0;
+ int num_nodes = 0,
+ num_leaves = 0;
+ dm_codes *codes,
+ *next_codes;
+ dm_node *nodes;
+ dm_leaves *leaves;
+
+ /* Convert input to encodable ASCII characters, stored in soundex buffer. */
+ normalize_input(word, soundex);
+ if (!soundex[0])
+ {
+ /* No encodable character in input. */
+ return NULL;
+ }
+
+ /* Allocate memory for node tree. */
+ nodes = palloc(sizeof(dm_nodes));
+ leaves = palloc(2 * sizeof(dm_leaves));
+
+ /* Starting point. */
+ nodes[num_nodes++] = start_node;
+ leaves[ix_leaves][num_leaves++] = &nodes[0];
+
+ codes = read_letter(soundex, &i);
+
+ while (codes)
+ {
+ next_codes = read_letter(soundex, &i);
+
+ /* Update leaf nodes. */
+ if (update_leaves(nodes, &num_nodes,
+ leaves, &ix_leaves, &num_leaves,
+ letter_no, codes, next_codes ? next_codes : end_codes))
+ {
+ /* All soundex codes are completed to six digits. */
+ break;
+ }
+
+ codes = next_codes;
+ letter_no++;
+ }
+
+ /* Concatenate all generated soundex codes. */
+ for (i = 0, j = 0;
+ i < num_leaves && j + DM_MAX_CODE_DIGITS + 1 <= n;
+ i++, j += DM_MAX_CODE_DIGITS + 1)
+ {
+ memcpy(&soundex[j], leaves[ix_leaves][i]->soundex, DM_MAX_CODE_DIGITS + 1);
+ }
+
+ /* Terminate string. */
+ soundex[j - (j != 0)] = '\0';
+
+ pfree(leaves);
+ pfree(nodes);
+
+ return soundex;
+}
diff --git a/contrib/fuzzystrmatch/daitch_mokotoff.h b/contrib/fuzzystrmatch/daitch_mokotoff.h
new file mode 100644
index 0000000000..8426069825
--- /dev/null
+++ b/contrib/fuzzystrmatch/daitch_mokotoff.h
@@ -0,0 +1,999 @@
+/*
+ * Types and lookup tables for Daitch-Mokotoff Soundex
+ *
+ * Copyright (c) 2021 Finance Norway
+ * Author: Dag Lem <[email protected]>
+ *
+ * This file is generated by daitch_mokotoff_header.pl
+ *
+ * Permission to use, copy, modify, and distribute this software and its
+ * documentation for any purpose, without fee, and without a written agreement
+ * is hereby granted, provided that the above copyright notice and this
+ * paragraph and the following two paragraphs appear in all copies.
+ *
+ * IN NO EVENT SHALL THE AUTHOR OR DISTRIBUTORS BE LIABLE TO ANY PARTY FOR
+ * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING
+ * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS
+ * DOCUMENTATION, EVEN IF THE AUTHOR OR DISTRIBUTORS HAVE BEEN ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ * THE AUTHOR AND DISTRIBUTORS SPECIFICALLY DISCLAIM ANY WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
+ * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS
+ * ON AN "AS IS" BASIS, AND THE AUTHOR AND DISTRIBUTORS HAS NO OBLIGATIONS TO
+ * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
+ */
+
+#include <stdlib.h>
+
+#define DM_MAX_CODE_DIGITS 6
+#define DM_MAX_ALTERNATE_CODES 5
+#define DM_MAX_NODES 1564
+#define DM_MAX_LEAVES 1250
+#define DM_MAX_SOUNDEX_CHARS (DM_MAX_NODES*(DM_MAX_CODE_DIGITS + 1))
+
+typedef char dm_code[2 + 1]; /* One or two sequential code digits + NUL */
+typedef dm_code dm_codes[3]; /* Start of name, before a vowel, any other */
+
+/* Letter in input sequence */
+struct dm_letter
+{
+ char letter; /* Present letter in sequence */
+ struct dm_letter *letters; /* List of possible successive letters */
+ dm_codes *codes; /* Code sequence(s) for complete sequence */
+};
+
+/* Node in soundex code tree */
+struct dm_node
+{
+ int soundex_length; /* Length of generated soundex code */
+ char soundex[DM_MAX_CODE_DIGITS + 1]; /* Soundex code */
+ int is_leaf; /* Candidate for complete soundex code */
+ int last_update; /* Letter number for last update of node */
+ char code_digit; /* Last code digit, 0 - 9 */
+
+ /*
+ * One or two alternate code digits leading to this node. If there are two
+ * digits, one of them is always an 'X'. Repeated code digits and 'X' lead
+ * back to the same node.
+ */
+ char prev_code_digits[2];
+ /* One or two alternate code digits moving forward. */
+ char next_code_digits[2];
+ /* ORed together code index(es) used to reach current node. */
+ int prev_code_index;
+ int next_code_index;
+ /* Nodes branching out from this node. */
+ struct dm_node *next_nodes[DM_MAX_ALTERNATE_CODES + 1];
+};
+
+typedef struct dm_letter dm_letter;
+typedef struct dm_node dm_node;
+
+/* Codes for letter sequence at start of name, before a vowel, and any other. */
+static dm_codes codes_0_1_X[2] =
+{
+ {
+ "0", "1", "X"
+ }
+};
+static dm_codes codes_0_7_X[2] =
+{
+ {
+ "0", "7", "X"
+ }
+};
+static dm_codes codes_0_X_X[2] =
+{
+ {
+ "0", "X", "X"
+ }
+};
+static dm_codes codes_1_1_X[2] =
+{
+ {
+ "1", "1", "X"
+ }
+};
+static dm_codes codes_1_X_X[2] =
+{
+ {
+ "1", "X", "X"
+ }
+};
+static dm_codes codes_1_X_X_or_4_4_4[2] =
+{
+ {
+ "1", "X", "X"
+ },
+ {
+ "4", "4", "4"
+ }
+};
+static dm_codes codes_2_43_43[2] =
+{
+ {
+ "2", "43", "43"
+ }
+};
+static dm_codes codes_2_4_4[2] =
+{
+ {
+ "2", "4", "4"
+ }
+};
+static dm_codes codes_3_3_3[2] =
+{
+ {
+ "3", "3", "3"
+ }
+};
+static dm_codes codes_3_3_3_or_4_4_4[2] =
+{
+ {
+ "3", "3", "3"
+ },
+ {
+ "4", "4", "4"
+ }
+};
+static dm_codes codes_4_4_4[2] =
+{
+ {
+ "4", "4", "4"
+ }
+};
+static dm_codes codes_5_54_54[2] =
+{
+ {
+ "5", "54", "54"
+ }
+};
+static dm_codes codes_5_5_5[2] =
+{
+ {
+ "5", "5", "5"
+ }
+};
+static dm_codes codes_5_5_5_or_45_45_45[2] =
+{
+ {
+ "5", "5", "5"
+ },
+ {
+ "45", "45", "45"
+ }
+};
+static dm_codes codes_5_5_5_or_4_4_4[2] =
+{
+ {
+ "5", "5", "5"
+ },
+ {
+ "4", "4", "4"
+ }
+};
+static dm_codes codes_5_5_X[2] =
+{
+ {
+ "5", "5", "X"
+ }
+};
+static dm_codes codes_66_66_66[2] =
+{
+ {
+ "66", "66", "66"
+ }
+};
+static dm_codes codes_6_6_6[2] =
+{
+ {
+ "6", "6", "6"
+ }
+};
+static dm_codes codes_7_7_7[2] =
+{
+ {
+ "7", "7", "7"
+ }
+};
+static dm_codes codes_8_8_8[2] =
+{
+ {
+ "8", "8", "8"
+ }
+};
+static dm_codes codes_94_94_94_or_4_4_4[2] =
+{
+ {
+ "94", "94", "94"
+ },
+ {
+ "4", "4", "4"
+ }
+};
+static dm_codes codes_9_9_9[2] =
+{
+ {
+ "9", "9", "9"
+ }
+};
+static dm_codes codes_X_X_6_or_X_X_X[2] =
+{
+ {
+ "X", "X", "6"
+ },
+ {
+ "X", "X", "X"
+ }
+};
+
+/* Coding for alternative following letters in sequence. */
+static dm_letter letter_A[] =
+{
+ {
+ 'I', NULL, codes_0_1_X
+ },
+ {
+ 'J', NULL, codes_0_1_X
+ },
+ {
+ 'U', NULL, codes_0_7_X
+ },
+ {
+ 'Y', NULL, codes_0_1_X
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_CH[] =
+{
+ {
+ 'S', NULL, codes_5_54_54
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_CS[] =
+{
+ {
+ 'Z', NULL, codes_4_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_CZ[] =
+{
+ {
+ 'S', NULL, codes_4_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_C[] =
+{
+ {
+ 'H', letter_CH, codes_5_5_5_or_4_4_4
+ },
+ {
+ 'K', NULL, codes_5_5_5_or_45_45_45
+ },
+ {
+ 'S', letter_CS, codes_4_4_4
+ },
+ {
+ 'Z', letter_CZ, codes_4_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_DR[] =
+{
+ {
+ 'S', NULL, codes_4_4_4
+ },
+ {
+ 'Z', NULL, codes_4_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_DS[] =
+{
+ {
+ 'H', NULL, codes_4_4_4
+ },
+ {
+ 'Z', NULL, codes_4_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_DZ[] =
+{
+ {
+ 'H', NULL, codes_4_4_4
+ },
+ {
+ 'S', NULL, codes_4_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_D[] =
+{
+ {
+ 'R', letter_DR, NULL
+ },
+ {
+ 'S', letter_DS, codes_4_4_4
+ },
+ {
+ 'T', NULL, codes_3_3_3
+ },
+ {
+ 'Z', letter_DZ, codes_4_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_E[] =
+{
+ {
+ 'I', NULL, codes_0_1_X
+ },
+ {
+ 'J', NULL, codes_0_1_X
+ },
+ {
+ 'U', NULL, codes_1_1_X
+ },
+ {
+ 'Y', NULL, codes_0_1_X
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_F[] =
+{
+ {
+ 'B', NULL, codes_7_7_7
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_I[] =
+{
+ {
+ 'A', NULL, codes_1_X_X
+ },
+ {
+ 'E', NULL, codes_1_X_X
+ },
+ {
+ 'O', NULL, codes_1_X_X
+ },
+ {
+ 'U', NULL, codes_1_X_X
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_K[] =
+{
+ {
+ 'H', NULL, codes_5_5_5
+ },
+ {
+ 'S', NULL, codes_5_54_54
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_M[] =
+{
+ {
+ 'N', NULL, codes_66_66_66
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_N[] =
+{
+ {
+ 'M', NULL, codes_66_66_66
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_O[] =
+{
+ {
+ 'I', NULL, codes_0_1_X
+ },
+ {
+ 'J', NULL, codes_0_1_X
+ },
+ {
+ 'Y', NULL, codes_0_1_X
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_P[] =
+{
+ {
+ 'F', NULL, codes_7_7_7
+ },
+ {
+ 'H', NULL, codes_7_7_7
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_R[] =
+{
+ {
+ 'S', NULL, codes_94_94_94_or_4_4_4
+ },
+ {
+ 'Z', NULL, codes_94_94_94_or_4_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_SCHTC[] =
+{
+ {
+ 'H', NULL, codes_2_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_SCHTSC[] =
+{
+ {
+ 'H', NULL, codes_2_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_SCHTS[] =
+{
+ {
+ 'C', letter_SCHTSC, NULL
+ },
+ {
+ 'H', NULL, codes_2_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_SCHT[] =
+{
+ {
+ 'C', letter_SCHTC, NULL
+ },
+ {
+ 'S', letter_SCHTS, NULL
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_SCH[] =
+{
+ {
+ 'D', NULL, codes_2_43_43
+ },
+ {
+ 'T', letter_SCHT, codes_2_43_43
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_SC[] =
+{
+ {
+ 'H', letter_SCH, codes_4_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_SHC[] =
+{
+ {
+ 'H', NULL, codes_2_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_SHTC[] =
+{
+ {
+ 'H', NULL, codes_2_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_SHTS[] =
+{
+ {
+ 'H', NULL, codes_2_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_SHT[] =
+{
+ {
+ 'C', letter_SHTC, NULL
+ },
+ {
+ 'S', letter_SHTS, NULL
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_SH[] =
+{
+ {
+ 'C', letter_SHC, NULL
+ },
+ {
+ 'D', NULL, codes_2_43_43
+ },
+ {
+ 'T', letter_SHT, codes_2_43_43
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_STC[] =
+{
+ {
+ 'H', NULL, codes_2_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_STR[] =
+{
+ {
+ 'S', NULL, codes_2_4_4
+ },
+ {
+ 'Z', NULL, codes_2_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_STSC[] =
+{
+ {
+ 'H', NULL, codes_2_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_STS[] =
+{
+ {
+ 'C', letter_STSC, NULL
+ },
+ {
+ 'H', NULL, codes_2_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_ST[] =
+{
+ {
+ 'C', letter_STC, NULL
+ },
+ {
+ 'R', letter_STR, NULL
+ },
+ {
+ 'S', letter_STS, NULL
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_SZC[] =
+{
+ {
+ 'S', NULL, codes_2_4_4
+ },
+ {
+ 'Z', NULL, codes_2_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_SZ[] =
+{
+ {
+ 'C', letter_SZC, NULL
+ },
+ {
+ 'D', NULL, codes_2_43_43
+ },
+ {
+ 'T', NULL, codes_2_43_43
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_S[] =
+{
+ {
+ 'C', letter_SC, codes_2_4_4
+ },
+ {
+ 'D', NULL, codes_2_43_43
+ },
+ {
+ 'H', letter_SH, codes_4_4_4
+ },
+ {
+ 'T', letter_ST, codes_2_43_43
+ },
+ {
+ 'Z', letter_SZ, codes_4_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_TC[] =
+{
+ {
+ 'H', NULL, codes_4_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_TR[] =
+{
+ {
+ 'S', NULL, codes_4_4_4
+ },
+ {
+ 'Z', NULL, codes_4_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_TSC[] =
+{
+ {
+ 'H', NULL, codes_4_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_TS[] =
+{
+ {
+ 'C', letter_TSC, NULL
+ },
+ {
+ 'H', NULL, codes_4_4_4
+ },
+ {
+ 'Z', NULL, codes_4_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_TTC[] =
+{
+ {
+ 'H', NULL, codes_4_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_TTSC[] =
+{
+ {
+ 'H', NULL, codes_4_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_TTS[] =
+{
+ {
+ 'C', letter_TTSC, NULL
+ },
+ {
+ 'Z', NULL, codes_4_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_TT[] =
+{
+ {
+ 'C', letter_TTC, NULL
+ },
+ {
+ 'S', letter_TTS, codes_4_4_4
+ },
+ {
+ 'Z', NULL, codes_4_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_TZ[] =
+{
+ {
+ 'S', NULL, codes_4_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_T[] =
+{
+ {
+ 'C', letter_TC, codes_4_4_4
+ },
+ {
+ 'H', NULL, codes_3_3_3
+ },
+ {
+ 'R', letter_TR, NULL
+ },
+ {
+ 'S', letter_TS, codes_4_4_4
+ },
+ {
+ 'T', letter_TT, NULL
+ },
+ {
+ 'Z', letter_TZ, codes_4_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_U[] =
+{
+ {
+ 'E', NULL, codes_0_1_X
+ },
+ {
+ 'I', NULL, codes_0_1_X
+ },
+ {
+ 'J', NULL, codes_0_1_X
+ },
+ {
+ 'Y', NULL, codes_0_1_X
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_ZDZ[] =
+{
+ {
+ 'H', NULL, codes_2_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_ZD[] =
+{
+ {
+ 'Z', letter_ZDZ, codes_2_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_ZHDZ[] =
+{
+ {
+ 'H', NULL, codes_2_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_ZHD[] =
+{
+ {
+ 'Z', letter_ZHDZ, NULL
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_ZH[] =
+{
+ {
+ 'D', letter_ZHD, codes_2_43_43
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_ZSC[] =
+{
+ {
+ 'H', NULL, codes_4_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_ZS[] =
+{
+ {
+ 'C', letter_ZSC, NULL
+ },
+ {
+ 'H', NULL, codes_4_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_Z[] =
+{
+ {
+ 'D', letter_ZD, codes_2_43_43
+ },
+ {
+ 'H', letter_ZH, codes_4_4_4
+ },
+ {
+ 'S', letter_ZS, codes_4_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_[] =
+{
+ {
+ 'A', letter_A, codes_0_X_X
+ },
+ {
+ 'B', NULL, codes_7_7_7
+ },
+ {
+ 'C', letter_C, codes_5_5_5_or_4_4_4
+ },
+ {
+ 'D', letter_D, codes_3_3_3
+ },
+ {
+ 'E', letter_E, codes_0_X_X
+ },
+ {
+ 'F', letter_F, codes_7_7_7
+ },
+ {
+ 'G', NULL, codes_5_5_5
+ },
+ {
+ 'H', NULL, codes_5_5_X
+ },
+ {
+ 'I', letter_I, codes_0_X_X
+ },
+ {
+ 'J', NULL, codes_1_X_X_or_4_4_4
+ },
+ {
+ 'K', letter_K, codes_5_5_5
+ },
+ {
+ 'L', NULL, codes_8_8_8
+ },
+ {
+ 'M', letter_M, codes_6_6_6
+ },
+ {
+ 'N', letter_N, codes_6_6_6
+ },
+ {
+ 'O', letter_O, codes_0_X_X
+ },
+ {
+ 'P', letter_P, codes_7_7_7
+ },
+ {
+ 'Q', NULL, codes_5_5_5
+ },
+ {
+ 'R', letter_R, codes_9_9_9
+ },
+ {
+ 'S', letter_S, codes_4_4_4
+ },
+ {
+ 'T', letter_T, codes_3_3_3
+ },
+ {
+ 'U', letter_U, codes_0_X_X
+ },
+ {
+ 'V', NULL, codes_7_7_7
+ },
+ {
+ 'W', NULL, codes_7_7_7
+ },
+ {
+ 'X', NULL, codes_5_54_54
+ },
+ {
+ 'Y', NULL, codes_1_X_X
+ },
+ {
+ 'Z', letter_Z, codes_4_4_4
+ },
+ {
+ 'a', NULL, codes_X_X_6_or_X_X_X
+ },
+ {
+ 'e', NULL, codes_X_X_6_or_X_X_X
+ },
+ {
+ 't', NULL, codes_3_3_3_or_4_4_4
+ },
+ {
+ '\0'
+ }
+};
diff --git a/contrib/fuzzystrmatch/daitch_mokotoff_header.pl b/contrib/fuzzystrmatch/daitch_mokotoff_header.pl
new file mode 100755
index 0000000000..3e97e000ee
--- /dev/null
+++ b/contrib/fuzzystrmatch/daitch_mokotoff_header.pl
@@ -0,0 +1,288 @@
+#!/bin/perl
+#
+# Generation of types and lookup tables for Daitch-Mokotoff soundex.
+#
+# Copyright (c) 2021 Finance Norway
+# Author: Dag Lem <[email protected]>
+#
+# Permission to use, copy, modify, and distribute this software and its
+# documentation for any purpose, without fee, and without a written agreement
+# is hereby granted, provided that the above copyright notice and this
+# paragraph and the following two paragraphs appear in all copies.
+#
+# IN NO EVENT SHALL THE AUTHOR OR DISTRIBUTORS BE LIABLE TO ANY PARTY FOR
+# DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING
+# LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS
+# DOCUMENTATION, EVEN IF THE AUTHOR OR DISTRIBUTORS HAVE BEEN ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+#
+# THE AUTHOR AND DISTRIBUTORS SPECIFICALLY DISCLAIM ANY WARRANTIES,
+# INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
+# AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS
+# ON AN "AS IS" BASIS, AND THE AUTHOR AND DISTRIBUTORS HAS NO OBLIGATIONS TO
+# PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
+#
+
+use strict;
+use warnings;
+use utf8;
+use open IO => ':utf8', ':std';
+use Data::Dumper;
+
+# Parse code table and generate tree for letter transitions.
+my %codes;
+my %alternates;
+my $table = [{}, [["","",""]]];
+while (<DATA>) {
+ chomp;
+ my ($letters, $codes) = split(/\s+/);
+ my @codes = map { [ split(/,/) ] } split(/\|/, $codes);
+
+ # Find alternate code transitions for calculation of storage.
+ # The first character can never yield more than two alternate codes, and is not considered here.
+ if (@codes > 1) {
+ for my $j (1..2) {
+ for my $i (0..1) {
+ my ($a, $b) = (substr($codes[$i][$j], -1, 1), substr($codes[($i + 1)%2][$j], 0, 1));
+ $alternates{$a}{$b} = 1 if $a ne $b;
+ }
+ }
+ }
+ my $key = "codes_" . join("_or_", map { join("_", @$_) } @codes);
+ my $val = join(",\n", map { "\t{\n\t\t" . join(", ", map { "\"$_\"" } @$_) . "\n\t}" } @codes);
+ $codes{$key} = $val;
+
+ for my $letter (split(/,/, $letters)) {
+ my $ref = $table->[0];
+ # Link each character to the next in the letter combination.
+ my @c = split(//, $letter);
+ my $last_c = pop(@c);
+ for my $c (@c) {
+ $ref->{$c} //= [ {}, undef ];
+ $ref->{$c}[0] //= {};
+ $ref = $ref->{$c}[0];
+ }
+ # The sound code for the letter combination is stored at the last character.
+ $ref->{$last_c}[1] = $key;
+ }
+}
+close(DATA);
+
+# Find the maximum number of alternate codes in one position.
+my $alt_x = $alternates{"X"};
+while (my ($k, $v) = each %alternates) {
+ if (defined delete $v->{"X"}) {
+ for my $x (keys %$alt_x) {
+ $v->{$x} = 1;
+ }
+ }
+}
+my $max_alt = (reverse sort (2, map { scalar keys %$_ } values %alternates))[0];
+
+# The maximum number of nodes and leaves in the soundex tree.
+# These are safe estimates, but in practice somewhat higher than the actual maximums.
+# Note that the first character can never yield more than two alternate codes,
+# hence the calculations are performed as sums of two subtrees.
+my $digits = 6;
+# Number of nodes (sum of geometric progression).
+my $max_nodes = 2 + 2*(1 - $max_alt**($digits - 1))/(1 - $max_alt);
+# Number of leaves (exponential of base number).
+my $max_leaves = 2*$max_alt**($digits - 2);
+
+print <<EOF;
+/*
+ * Types and lookup tables for Daitch-Mokotoff Soundex
+ *
+ * Copyright (c) 2021 Finance Norway
+ * Author: Dag Lem <dag\@nimrod.no>
+ *
+ * This file is generated by daitch_mokotoff_header.pl
+ *
+ * Permission to use, copy, modify, and distribute this software and its
+ * documentation for any purpose, without fee, and without a written agreement
+ * is hereby granted, provided that the above copyright notice and this
+ * paragraph and the following two paragraphs appear in all copies.
+ *
+ * IN NO EVENT SHALL THE AUTHOR OR DISTRIBUTORS BE LIABLE TO ANY PARTY FOR
+ * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING
+ * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS
+ * DOCUMENTATION, EVEN IF THE AUTHOR OR DISTRIBUTORS HAVE BEEN ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ * THE AUTHOR AND DISTRIBUTORS SPECIFICALLY DISCLAIM ANY WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
+ * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS
+ * ON AN "AS IS" BASIS, AND THE AUTHOR AND DISTRIBUTORS HAS NO OBLIGATIONS TO
+ * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
+ */
+
+#include <stdlib.h>
+
+#define DM_MAX_CODE_DIGITS $digits
+#define DM_MAX_ALTERNATE_CODES $max_alt
+#define DM_MAX_NODES $max_nodes
+#define DM_MAX_LEAVES $max_leaves
+#define DM_MAX_SOUNDEX_CHARS (DM_MAX_NODES*(DM_MAX_CODE_DIGITS + 1))
+
+typedef char dm_code[2 + 1]; /* One or two sequential code digits + NUL */
+typedef dm_code dm_codes[3]; /* Start of name, before a vowel, any other */
+
+/* Letter in input sequence */
+struct dm_letter
+{
+ char letter; /* Present letter in sequence */
+ struct dm_letter *letters; /* List of possible successive letters */
+ dm_codes *codes; /* Code sequence(s) for complete sequence */
+};
+
+/* Node in soundex code tree */
+struct dm_node
+{
+ int soundex_length; /* Length of generated soundex code */
+ char soundex[DM_MAX_CODE_DIGITS + 1]; /* Soundex code */
+ int is_leaf; /* Candidate for complete soundex code */
+ int last_update; /* Letter number for last update of node */
+ char code_digit; /* Last code digit, 0 - 9 */
+
+ /*
+ * One or two alternate code digits leading to this node. If there are two
+ * digits, one of them is always an 'X'. Repeated code digits and 'X' lead
+ * back to the same node.
+ */
+ char prev_code_digits[2];
+ /* One or two alternate code digits moving forward. */
+ char next_code_digits[2];
+ /* ORed together code index(es) used to reach current node. */
+ int prev_code_index;
+ int next_code_index;
+ /* Nodes branching out from this node. */
+ struct dm_node *next_nodes[DM_MAX_ALTERNATE_CODES + 1];
+};
+
+typedef struct dm_letter dm_letter;
+typedef struct dm_node dm_node;
+
+/* Codes for letter sequence at start of name, before a vowel, and any other. */
+EOF
+
+for my $key (sort keys %codes) {
+ print "static dm_codes $key\[2\] =\n{\n" . $codes{$key} . "\n};\n";
+}
+
+print <<EOF;
+
+/* Coding for alternative following letters in sequence. */
+EOF
+
+sub hash2code {
+ my ($ref, $letter) = @_;
+
+ my @letters = ();
+
+ my $h = $ref->[0];
+ for my $key (sort keys %$h) {
+ $ref = $h->{$key};
+ my $children = "NULL";
+ if (defined $ref->[0]) {
+ $children = "letter_$letter$key";
+ hash2code($ref, "$letter$key");
+ }
+ my $codes = $ref->[1] // "NULL";
+ push(@letters, "\t{\n\t\t'$key', $children, $codes\n\t}");
+ }
+
+ print "static dm_letter letter_$letter\[\] =\n{\n";
+ for (@letters) {
+ print "$_,\n";
+ }
+ print "\t{\n\t\t'\\0'\n\t}\n";
+ print "};\n";
+}
+
+hash2code($table, '');
+
+
+# Table adapted from https://www.jewishgen.org/InfoFiles/Soundex.html
+#
+# X = NC (not coded)
+#
+# Note that the following letters are coded with substitute letters
+#
+# Ą => a (use '[' for table lookup)
+# Ę => e (use '\\' for table lookup)
+# Ţ => t (use ']' for table lookup)
+#
+# The rule for "UE" below does not correspond to the table referred to above,
+# however it is used by all other known implementations, including the one at
+# https://www.jewishgen.org/jos/jossound.htm (try e.g. "bouey").
+
+__DATA__
+AI,AJ,AY 0,1,X
+AU 0,7,X
+a X,X,6|X,X,X
+A 0,X,X
+B 7,7,7
+CHS 5,54,54
+CH 5,5,5|4,4,4
+CK 5,5,5|45,45,45
+CZ,CS,CSZ,CZS 4,4,4
+C 5,5,5|4,4,4
+DRZ,DRS 4,4,4
+DS,DSH,DSZ 4,4,4
+DZ,DZH,DZS 4,4,4
+D,DT 3,3,3
+EI,EJ,EY 0,1,X
+EU 1,1,X
+e X,X,6|X,X,X
+E 0,X,X
+FB 7,7,7
+F 7,7,7
+G 5,5,5
+H 5,5,X
+IA,IE,IO,IU 1,X,X
+I 0,X,X
+J 1,X,X|4,4,4
+KS 5,54,54
+KH 5,5,5
+K 5,5,5
+L 8,8,8
+MN 66,66,66
+M 6,6,6
+NM 66,66,66
+N 6,6,6
+OI,OJ,OY 0,1,X
+O 0,X,X
+P,PF,PH 7,7,7
+Q 5,5,5
+RZ,RS 94,94,94|4,4,4
+R 9,9,9
+SCHTSCH,SCHTSH,SCHTCH 2,4,4
+SCH 4,4,4
+SHTCH,SHCH,SHTSH 2,4,4
+SHT,SCHT,SCHD 2,43,43
+SH 4,4,4
+STCH,STSCH,SC 2,4,4
+STRZ,STRS,STSH 2,4,4
+ST 2,43,43
+SZCZ,SZCS 2,4,4
+SZT,SHD,SZD,SD 2,43,43
+SZ 4,4,4
+S 4,4,4
+TCH,TTCH,TTSCH 4,4,4
+TH 3,3,3
+TRZ,TRS 4,4,4
+TSCH,TSH 4,4,4
+TS,TTS,TTSZ,TC 4,4,4
+TZ,TTZ,TZS,TSZ 4,4,4
+t 3,3,3|4,4,4
+T 3,3,3
+UI,UJ,UY,UE 0,1,X
+U 0,X,X
+V 7,7,7
+W 7,7,7
+X 5,54,54
+Y 1,X,X
+ZDZ,ZDZH,ZHDZH 2,4,4
+ZD,ZHD 2,43,43
+ZH,ZS,ZSCH,ZSH 4,4,4
+Z 4,4,4
diff --git a/contrib/fuzzystrmatch/expected/fuzzystrmatch.out b/contrib/fuzzystrmatch/expected/fuzzystrmatch.out
index 493c95cdfa..f62ddad4ee 100644
--- a/contrib/fuzzystrmatch/expected/fuzzystrmatch.out
+++ b/contrib/fuzzystrmatch/expected/fuzzystrmatch.out
@@ -65,3 +65,174 @@ SELECT dmetaphone_alt('gumbo');
KMP
(1 row)
+-- Wovels
+SELECT daitch_mokotoff('Augsburg');
+ daitch_mokotoff
+-----------------
+ 054795
+(1 row)
+
+SELECT daitch_mokotoff('Breuer');
+ daitch_mokotoff
+-----------------
+ 791900
+(1 row)
+
+SELECT daitch_mokotoff('Freud');
+ daitch_mokotoff
+-----------------
+ 793000
+(1 row)
+
+-- The letter "H"
+SELECT daitch_mokotoff('Halberstadt');
+ daitch_mokotoff
+-----------------
+ 587943 587433
+(1 row)
+
+SELECT daitch_mokotoff('Mannheim');
+ daitch_mokotoff
+-----------------
+ 665600
+(1 row)
+
+-- Adjacent sounds
+SELECT daitch_mokotoff('Chernowitz');
+ daitch_mokotoff
+-----------------
+ 596740 496740
+(1 row)
+
+-- Adjacent letters with identical adjacent code digits
+SELECT daitch_mokotoff('Cherkassy');
+ daitch_mokotoff
+-----------------
+ 595400 495400
+(1 row)
+
+SELECT daitch_mokotoff('Kleinman');
+ daitch_mokotoff
+-----------------
+ 586660
+(1 row)
+
+-- More than one word
+SELECT daitch_mokotoff('Nowy Targ');
+ daitch_mokotoff
+-----------------
+ 673950
+(1 row)
+
+-- Padded with "0"
+SELECT daitch_mokotoff('Berlin');
+ daitch_mokotoff
+-----------------
+ 798600
+(1 row)
+
+-- Other examples from https://www.avotaynu.com/soundex.htm
+SELECT daitch_mokotoff('Ceniow');
+ daitch_mokotoff
+-----------------
+ 567000 467000
+(1 row)
+
+SELECT daitch_mokotoff('Tsenyuv');
+ daitch_mokotoff
+-----------------
+ 467000
+(1 row)
+
+SELECT daitch_mokotoff('Holubica');
+ daitch_mokotoff
+-----------------
+ 587500 587400
+(1 row)
+
+SELECT daitch_mokotoff('Golubitsa');
+ daitch_mokotoff
+-----------------
+ 587400
+(1 row)
+
+SELECT daitch_mokotoff('Przemysl');
+ daitch_mokotoff
+-----------------
+ 794648 746480
+(1 row)
+
+SELECT daitch_mokotoff('Pshemeshil');
+ daitch_mokotoff
+-----------------
+ 746480
+(1 row)
+
+SELECT daitch_mokotoff('Rosochowaciec');
+ daitch_mokotoff
+---------------------------------------------------------
+ 945755 945754 945745 945744 944755 944754 944745 944744
+(1 row)
+
+SELECT daitch_mokotoff('Rosokhovatsets');
+ daitch_mokotoff
+-----------------
+ 945744
+(1 row)
+
+-- Ignored characters
+SELECT daitch_mokotoff('''OBrien');
+ daitch_mokotoff
+-----------------
+ 079600
+(1 row)
+
+SELECT daitch_mokotoff('O''Brien');
+ daitch_mokotoff
+-----------------
+ 079600
+(1 row)
+
+-- "Difficult" cases, likely to cause trouble for other implementations.
+SELECT daitch_mokotoff('CJC');
+ daitch_mokotoff
+-------------------------------------------
+ 550000 540000 545000 450000 400000 440000
+(1 row)
+
+SELECT daitch_mokotoff('BESST');
+ daitch_mokotoff
+-----------------
+ 743000
+(1 row)
+
+SELECT daitch_mokotoff('BOUEY');
+ daitch_mokotoff
+-----------------
+ 710000
+(1 row)
+
+SELECT daitch_mokotoff('HANNMANN');
+ daitch_mokotoff
+-----------------
+ 566600
+(1 row)
+
+SELECT daitch_mokotoff('MCCOYJR');
+ daitch_mokotoff
+---------------------------------------------------------
+ 651900 654900 654190 654490 645190 645490 641900 644900
+(1 row)
+
+SELECT daitch_mokotoff('ACCURSO');
+ daitch_mokotoff
+---------------------------------------------------------
+ 059400 054000 054940 054400 045940 045400 049400 044000
+(1 row)
+
+SELECT daitch_mokotoff('BIERSCHBACH');
+ daitch_mokotoff
+---------------------------------------------------------
+ 794575 794574 794750 794740 745750 745740 747500 747400
+(1 row)
+
diff --git a/contrib/fuzzystrmatch/expected/fuzzystrmatch_utf8.out b/contrib/fuzzystrmatch/expected/fuzzystrmatch_utf8.out
new file mode 100644
index 0000000000..32d8260383
--- /dev/null
+++ b/contrib/fuzzystrmatch/expected/fuzzystrmatch_utf8.out
@@ -0,0 +1,61 @@
+/*
+ * This test must be run in a database with UTF-8 encoding,
+ * because other encodings don't support all the characters used.
+ */
+SELECT getdatabaseencoding() <> 'UTF8'
+ AS skip_test \gset
+\if :skip_test
+\quit
+\endif
+set client_encoding = utf8;
+-- CREATE EXTENSION IF NOT EXISTS fuzzystrmatch;
+-- Accents
+SELECT daitch_mokotoff('Müller');
+ daitch_mokotoff
+-----------------
+ 689000
+(1 row)
+
+SELECT daitch_mokotoff('Schäfer');
+ daitch_mokotoff
+-----------------
+ 479000
+(1 row)
+
+SELECT daitch_mokotoff('Straßburg');
+ daitch_mokotoff
+-----------------
+ 294795
+(1 row)
+
+SELECT daitch_mokotoff('Éregon');
+ daitch_mokotoff
+-----------------
+ 095600
+(1 row)
+
+-- Special characters added at https://www.jewishgen.org/InfoFiles/Soundex.html
+SELECT daitch_mokotoff('gąszczu');
+ daitch_mokotoff
+-----------------
+ 564000 540000
+(1 row)
+
+SELECT daitch_mokotoff('brzęczy');
+ daitch_mokotoff
+-----------------------------
+ 794640 794400 746400 744000
+(1 row)
+
+SELECT daitch_mokotoff('ţamas');
+ daitch_mokotoff
+-----------------
+ 364000 464000
+(1 row)
+
+SELECT daitch_mokotoff('țamas');
+ daitch_mokotoff
+-----------------
+ 364000 464000
+(1 row)
+
diff --git a/contrib/fuzzystrmatch/expected/fuzzystrmatch_utf8_1.out b/contrib/fuzzystrmatch/expected/fuzzystrmatch_utf8_1.out
new file mode 100644
index 0000000000..37aead89c0
--- /dev/null
+++ b/contrib/fuzzystrmatch/expected/fuzzystrmatch_utf8_1.out
@@ -0,0 +1,8 @@
+/*
+ * This test must be run in a database with UTF-8 encoding,
+ * because other encodings don't support all the characters used.
+ */
+SELECT getdatabaseencoding() <> 'UTF8'
+ AS skip_test \gset
+\if :skip_test
+\quit
diff --git a/contrib/fuzzystrmatch/fuzzystrmatch--1.1--1.2.sql b/contrib/fuzzystrmatch/fuzzystrmatch--1.1--1.2.sql
new file mode 100644
index 0000000000..b9d7b229a3
--- /dev/null
+++ b/contrib/fuzzystrmatch/fuzzystrmatch--1.1--1.2.sql
@@ -0,0 +1,8 @@
+/* contrib/fuzzystrmatch/fuzzystrmatch--1.1--1.2.sql */
+
+-- complain if script is sourced in psql, rather than via ALTER EXTENSION
+\echo Use "ALTER EXTENSION fuzzystrmatch UPDATE TO '1.2'" to load this file. \quit
+
+CREATE FUNCTION daitch_mokotoff(text) RETURNS text
+AS 'MODULE_PATHNAME', 'daitch_mokotoff'
+LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
diff --git a/contrib/fuzzystrmatch/fuzzystrmatch--1.1.sql b/contrib/fuzzystrmatch/fuzzystrmatch--1.2.sql
similarity index 92%
rename from contrib/fuzzystrmatch/fuzzystrmatch--1.1.sql
rename to contrib/fuzzystrmatch/fuzzystrmatch--1.2.sql
index 41de9d949b..2a8a100699 100644
--- a/contrib/fuzzystrmatch/fuzzystrmatch--1.1.sql
+++ b/contrib/fuzzystrmatch/fuzzystrmatch--1.2.sql
@@ -42,3 +42,7 @@ LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION dmetaphone_alt (text) RETURNS text
AS 'MODULE_PATHNAME', 'dmetaphone_alt'
LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
+
+CREATE FUNCTION daitch_mokotoff(text) RETURNS text
+AS 'MODULE_PATHNAME', 'daitch_mokotoff'
+LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
diff --git a/contrib/fuzzystrmatch/fuzzystrmatch.control b/contrib/fuzzystrmatch/fuzzystrmatch.control
index 3cd6660bf9..8b6e9fd993 100644
--- a/contrib/fuzzystrmatch/fuzzystrmatch.control
+++ b/contrib/fuzzystrmatch/fuzzystrmatch.control
@@ -1,6 +1,6 @@
# fuzzystrmatch extension
comment = 'determine similarities and distance between strings'
-default_version = '1.1'
+default_version = '1.2'
module_pathname = '$libdir/fuzzystrmatch'
relocatable = true
trusted = true
diff --git a/contrib/fuzzystrmatch/sql/fuzzystrmatch.sql b/contrib/fuzzystrmatch/sql/fuzzystrmatch.sql
index f05dc28ffb..db05c7d6b6 100644
--- a/contrib/fuzzystrmatch/sql/fuzzystrmatch.sql
+++ b/contrib/fuzzystrmatch/sql/fuzzystrmatch.sql
@@ -19,3 +19,48 @@ SELECT metaphone('GUMBO', 4);
SELECT dmetaphone('gumbo');
SELECT dmetaphone_alt('gumbo');
+
+-- Wovels
+SELECT daitch_mokotoff('Augsburg');
+SELECT daitch_mokotoff('Breuer');
+SELECT daitch_mokotoff('Freud');
+
+-- The letter "H"
+SELECT daitch_mokotoff('Halberstadt');
+SELECT daitch_mokotoff('Mannheim');
+
+-- Adjacent sounds
+SELECT daitch_mokotoff('Chernowitz');
+
+-- Adjacent letters with identical adjacent code digits
+SELECT daitch_mokotoff('Cherkassy');
+SELECT daitch_mokotoff('Kleinman');
+
+-- More than one word
+SELECT daitch_mokotoff('Nowy Targ');
+
+-- Padded with "0"
+SELECT daitch_mokotoff('Berlin');
+
+-- Other examples from https://www.avotaynu.com/soundex.htm
+SELECT daitch_mokotoff('Ceniow');
+SELECT daitch_mokotoff('Tsenyuv');
+SELECT daitch_mokotoff('Holubica');
+SELECT daitch_mokotoff('Golubitsa');
+SELECT daitch_mokotoff('Przemysl');
+SELECT daitch_mokotoff('Pshemeshil');
+SELECT daitch_mokotoff('Rosochowaciec');
+SELECT daitch_mokotoff('Rosokhovatsets');
+
+-- Ignored characters
+SELECT daitch_mokotoff('''OBrien');
+SELECT daitch_mokotoff('O''Brien');
+
+-- "Difficult" cases, likely to cause trouble for other implementations.
+SELECT daitch_mokotoff('CJC');
+SELECT daitch_mokotoff('BESST');
+SELECT daitch_mokotoff('BOUEY');
+SELECT daitch_mokotoff('HANNMANN');
+SELECT daitch_mokotoff('MCCOYJR');
+SELECT daitch_mokotoff('ACCURSO');
+SELECT daitch_mokotoff('BIERSCHBACH');
diff --git a/contrib/fuzzystrmatch/sql/fuzzystrmatch_utf8.sql b/contrib/fuzzystrmatch/sql/fuzzystrmatch_utf8.sql
new file mode 100644
index 0000000000..f42c01a1bb
--- /dev/null
+++ b/contrib/fuzzystrmatch/sql/fuzzystrmatch_utf8.sql
@@ -0,0 +1,26 @@
+/*
+ * This test must be run in a database with UTF-8 encoding,
+ * because other encodings don't support all the characters used.
+ */
+
+SELECT getdatabaseencoding() <> 'UTF8'
+ AS skip_test \gset
+\if :skip_test
+\quit
+\endif
+
+set client_encoding = utf8;
+
+-- CREATE EXTENSION IF NOT EXISTS fuzzystrmatch;
+
+-- Accents
+SELECT daitch_mokotoff('Müller');
+SELECT daitch_mokotoff('Schäfer');
+SELECT daitch_mokotoff('Straßburg');
+SELECT daitch_mokotoff('Éregon');
+
+-- Special characters added at https://www.jewishgen.org/InfoFiles/Soundex.html
+SELECT daitch_mokotoff('gąszczu');
+SELECT daitch_mokotoff('brzęczy');
+SELECT daitch_mokotoff('ţamas');
+SELECT daitch_mokotoff('țamas');
diff --git a/doc/src/sgml/fuzzystrmatch.sgml b/doc/src/sgml/fuzzystrmatch.sgml
index 382e54be91..08781778f8 100644
--- a/doc/src/sgml/fuzzystrmatch.sgml
+++ b/doc/src/sgml/fuzzystrmatch.sgml
@@ -241,4 +241,101 @@ test=# SELECT dmetaphone('gumbo');
</screen>
</sect2>
+ <sect2>
+ <title>Daitch-Mokotoff Soundex</title>
+
+ <para>
+ Compared to the American Soundex System implemented in the
+ <function>soundex</function> function, the major improvements of the
+ Daitch-Mokotoff Soundex System are:
+
+ <itemizedlist spacing="compact" mark="bullet">
+ <listitem>
+ <para>
+ Information is coded to the first six meaningful letters rather than
+ four.
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ The initial letter is coded rather than kept as is.
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ Where two consecutive letters have a single sound, they are coded as a
+ single number.
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ When a letter or combination of letters may have two different sounds,
+ it is double coded under the two different codes.
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ A letter or combination of letters maps into ten possible codes rather
+ than seven.
+ </para>
+ </listitem>
+ </itemizedlist>
+ </para>
+
+ <indexterm>
+ <primary>daitch_mokotoff</primary>
+ </indexterm>
+
+ <para>
+ The following function generates Daitch-Mokotoff soundex codes for matching
+ of similar-sounding input:
+ </para>
+
+<synopsis>
+daitch_mokotoff(text source) returns text
+</synopsis>
+
+ <para>
+ Since a Daitch-Mokotoff soundex code consists of only 6 digits,
+ <literal>source</literal> should be preferably a single word or name.
+ Any alternative soundex codes are separated by space, which makes the returned
+ text suited for use in Full Text Search, see <xref linkend="textsearch"/> and
+ <xref linkend="functions-textsearch"/>.
+ </para>
+
+ <para>
+ Example:
+ </para>
+
+<programlisting>
+CREATE OR REPLACE FUNCTION soundex_name(v_name text) RETURNS text AS $$
+ SELECT string_agg(daitch_mokotoff(n), ' ')
+ FROM regexp_split_to_table(v_name, '\s+') AS n
+$$ LANGUAGE sql STRICT IMMUTABLE PARALLEL SAFE;
+
+CREATE OR REPLACE FUNCTION soundex_tsvector(v_name text) RETURNS tsvector AS $$
+ SELECT to_tsvector('simple', soundex_name(v_name))
+$$ LANGUAGE sql STRICT IMMUTABLE PARALLEL SAFE;
+
+CREATE OR REPLACE FUNCTION soundex_tsquery(v_name text) RETURNS tsquery AS $$
+ SELECT to_tsquery('simple', quote_literal(soundex_name(v_name)))
+$$ LANGUAGE sql STRICT IMMUTABLE PARALLEL SAFE;
+
+-- Note that searches could be more efficient with the tsvector in a separate column
+-- (no recalculation on table row recheck).
+CREATE TABLE s (nm text);
+CREATE INDEX ix_s_txt ON s USING gin (soundex_tsvector(nm)) WITH (fastupdate = off);
+
+INSERT INTO s VALUES ('John Doe');
+INSERT INTO s VALUES ('Jane Roe');
+INSERT INTO s VALUES ('Public John Q.');
+INSERT INTO s VALUES ('George Best');
+
+SELECT * FROM s WHERE soundex_tsvector(nm) @@ soundex_tsquery('john');
+SELECT * FROM s WHERE soundex_tsvector(nm) @@ soundex_tsquery('jane doe');
+SELECT * FROM s WHERE soundex_tsvector(nm) @@ soundex_tsquery('john public');
+SELECT * FROM s WHERE soundex_tsvector(nm) @@ soundex_tsquery('besst, giorgio');
+</programlisting>
+ </sect2>
+
</sect1>
^ permalink raw reply [nested|flat] 19+ messages in thread
* Re: daitch_mokotoff module
2022-01-03 19:16 Re: daitch_mokotoff module Andres Freund <[email protected]>
2022-01-04 13:49 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
@ 2022-01-04 20:18 ` Thomas Munro <[email protected]>
2022-01-05 07:05 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
0 siblings, 1 reply; 19+ messages in thread
From: Thomas Munro @ 2022-01-04 20:18 UTC (permalink / raw)
To: Dag Lem <[email protected]>; +Cc: Andres Freund <[email protected]>; Tom Lane <[email protected]>; PostgreSQL Hackers <[email protected]>
On Wed, Jan 5, 2022 at 2:49 AM Dag Lem <[email protected]> wrote:
> However I guess this won't make any difference wrt. actually running the
> tests, as long as there seems to be an encoding problem in the cfbot
Fixed -- I told it to pull down patches as binary, not text. Now it
makes commits that look healthier, and so far all the Unix systems
have survived CI:
https://github.com/postgresql-cfbot/postgresql/commit/79700efc61d15c2414b8450a786951fa9308c07f
http://cfbot.cputube.org/dag-lem.html
^ permalink raw reply [nested|flat] 19+ messages in thread
* Re: daitch_mokotoff module
2022-01-03 19:16 Re: daitch_mokotoff module Andres Freund <[email protected]>
2022-01-04 13:49 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
2022-01-04 20:18 ` Re: daitch_mokotoff module Thomas Munro <[email protected]>
@ 2022-01-05 07:05 ` Dag Lem <[email protected]>
2022-01-05 20:08 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
0 siblings, 1 reply; 19+ messages in thread
From: Dag Lem @ 2022-01-05 07:05 UTC (permalink / raw)
To: Thomas Munro <[email protected]>; +Cc: Andres Freund <[email protected]>; Tom Lane <[email protected]>; PostgreSQL Hackers <[email protected]>
Thomas Munro <[email protected]> writes:
> On Wed, Jan 5, 2022 at 2:49 AM Dag Lem <[email protected]> wrote:
>> However I guess this won't make any difference wrt. actually running the
>> tests, as long as there seems to be an encoding problem in the cfbot
>
> Fixed -- I told it to pull down patches as binary, not text. Now it
> makes commits that look healthier, and so far all the Unix systems
> have survived CI:
>
> https://github.com/postgresql-cfbot/postgresql/commit/79700efc61d15c2414b8450a786951fa9308c07f
> http://cfbot.cputube.org/dag-lem.html
>
Great!
Dag
^ permalink raw reply [nested|flat] 19+ messages in thread
* Re: daitch_mokotoff module
2022-01-03 19:16 Re: daitch_mokotoff module Andres Freund <[email protected]>
2022-01-04 13:49 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
2022-01-04 20:18 ` Re: daitch_mokotoff module Thomas Munro <[email protected]>
2022-01-05 07:05 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
@ 2022-01-05 20:08 ` Dag Lem <[email protected]>
2022-02-03 14:27 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
0 siblings, 1 reply; 19+ messages in thread
From: Dag Lem @ 2022-01-05 20:08 UTC (permalink / raw)
To: Thomas Munro <[email protected]>; +Cc: Andres Freund <[email protected]>; Tom Lane <[email protected]>; PostgreSQL Hackers <[email protected]>
Dag Lem <[email protected]> writes:
> Thomas Munro <[email protected]> writes:
>
>> On Wed, Jan 5, 2022 at 2:49 AM Dag Lem <[email protected]> wrote:
>>> However I guess this won't make any difference wrt. actually running the
>>> tests, as long as there seems to be an encoding problem in the cfbot
>>
>> Fixed -- I told it to pull down patches as binary, not text. Now it
>> makes commits that look healthier, and so far all the Unix systems
>> have survived CI:
>>
>> https://github.com/postgresql-cfbot/postgresql/commit/79700efc61d15c2414b8450a786951fa9308c07f
>> http://cfbot.cputube.org/dag-lem.html
>>
>
> Great!
>
> Dag
>
>
After this I did the mistake of including a patch for citext in this
thread, which is now picked up by cfbot instead of the Daitch-Mokotoff
patch.
Attaching the original patch again in order to hopefully fix my mistake.
Best regards
Dag Lem
Attachments:
[text/x-patch] v5-daitch_mokotoff.patch (51.2K, ../../[email protected]/2-v5-daitch_mokotoff.patch)
download | inline diff:
diff --git a/contrib/fuzzystrmatch/Makefile b/contrib/fuzzystrmatch/Makefile
index 0704894f88..1d5bd84be8 100644
--- a/contrib/fuzzystrmatch/Makefile
+++ b/contrib/fuzzystrmatch/Makefile
@@ -3,14 +3,15 @@
MODULE_big = fuzzystrmatch
OBJS = \
$(WIN32RES) \
+ daitch_mokotoff.o \
dmetaphone.o \
fuzzystrmatch.o
EXTENSION = fuzzystrmatch
-DATA = fuzzystrmatch--1.1.sql fuzzystrmatch--1.0--1.1.sql
+DATA = fuzzystrmatch--1.2.sql fuzzystrmatch--1.1--1.2.sql fuzzystrmatch--1.0--1.1.sql
PGFILEDESC = "fuzzystrmatch - similarities and distance between strings"
-REGRESS = fuzzystrmatch
+REGRESS = fuzzystrmatch fuzzystrmatch_utf8
ifdef USE_PGXS
PG_CONFIG = pg_config
@@ -22,3 +23,8 @@ top_builddir = ../..
include $(top_builddir)/src/Makefile.global
include $(top_srcdir)/contrib/contrib-global.mk
endif
+
+daitch_mokotoff.o: daitch_mokotoff.h
+
+daitch_mokotoff.h: daitch_mokotoff_header.pl
+ perl $< > $@
diff --git a/contrib/fuzzystrmatch/daitch_mokotoff.c b/contrib/fuzzystrmatch/daitch_mokotoff.c
new file mode 100644
index 0000000000..1b7263c349
--- /dev/null
+++ b/contrib/fuzzystrmatch/daitch_mokotoff.c
@@ -0,0 +1,593 @@
+/*
+ * Daitch-Mokotoff Soundex
+ *
+ * Copyright (c) 2021 Finance Norway
+ * Author: Dag Lem <[email protected]>
+ *
+ * This implementation of the Daitch-Mokotoff Soundex System aims at high
+ * performance.
+ *
+ * - The processing of each phoneme is initiated by an O(1) table lookup.
+ * - For phonemes containing more than one character, a coding tree is traversed
+ * to process the complete phoneme.
+ * - The (alternate) soundex codes are produced digit by digit in-place in
+ * another tree structure.
+ *
+ * References:
+ *
+ * https://www.avotaynu.com/soundex.htm
+ * https://www.jewishgen.org/InfoFiles/Soundex.html
+ * https://familypedia.fandom.com/wiki/Daitch-Mokotoff_Soundex
+ * https://stevemorse.org/census/soundex.html (dmlat.php, dmsoundex.php)
+ * https://github.com/apache/commons-codec/ (dmrules.txt, DaitchMokotoffSoundex.java)
+ * https://metacpan.org/pod/Text::Phonetic (DaitchMokotoff.pm)
+ *
+ * A few notes on other implementations:
+ *
+ * - All other known implementations have the same unofficial rules for "UE",
+ * these are also adapted by this implementation (0, 1, NC).
+ * - The only other known implementation which is capable of generating all
+ * correct soundex codes in all cases is the JOS Soundex Calculator at
+ * https://www.jewishgen.org/jos/jossound.htm
+ * - "J" is considered (only) a vowel in dmlat.php
+ * - The official rules for "RS" are commented out in dmlat.php
+ * - Identical code digits for adjacent letters are not collapsed correctly in
+ * dmsoundex.php when double digit codes are involved. E.g. "BESST" yields
+ * 744300 instead of 743000 as for "BEST".
+ * - "J" is considered (only) a consonant in DaitchMokotoffSoundex.java
+ * - "Y" is not considered a vowel in DaitchMokotoffSoundex.java
+ *
+ * Permission to use, copy, modify, and distribute this software and its
+ * documentation for any purpose, without fee, and without a written agreement
+ * is hereby granted, provided that the above copyright notice and this
+ * paragraph and the following two paragraphs appear in all copies.
+ *
+ * IN NO EVENT SHALL THE AUTHOR OR DISTRIBUTORS BE LIABLE TO ANY PARTY FOR
+ * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING
+ * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS
+ * DOCUMENTATION, EVEN IF THE AUTHOR OR DISTRIBUTORS HAVE BEEN ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ * THE AUTHOR AND DISTRIBUTORS SPECIFICALLY DISCLAIM ANY WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
+ * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS
+ * ON AN "AS IS" BASIS, AND THE AUTHOR AND DISTRIBUTORS HAS NO OBLIGATIONS TO
+ * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
+*/
+
+#include "daitch_mokotoff.h"
+
+#include "postgres.h"
+#include "utils/builtins.h"
+#include "mb/pg_wchar.h"
+
+#include <ctype.h>
+#include <string.h>
+
+/* Internal C implementation */
+static char *_daitch_mokotoff(char *word, char *soundex, size_t n);
+
+
+PG_FUNCTION_INFO_V1(daitch_mokotoff);
+Datum
+daitch_mokotoff(PG_FUNCTION_ARGS)
+{
+ text *arg = PG_GETARG_TEXT_PP(0);
+ char *string,
+ *tmp_soundex;
+ text *soundex;
+
+ /*
+ * The maximum theoretical soundex size is several KB, however in practice
+ * anything but contrived synthetic inputs will yield a soundex size of
+ * less than 100 bytes. We thus allocate and free a temporary work buffer,
+ * and return only the actual soundex result.
+ */
+ string = pg_server_to_any(text_to_cstring(arg), VARSIZE_ANY_EXHDR(arg), PG_UTF8);
+ tmp_soundex = palloc(DM_MAX_SOUNDEX_CHARS);
+
+ if (!_daitch_mokotoff(string, tmp_soundex, DM_MAX_SOUNDEX_CHARS))
+ {
+ /* No encodable characters in input. */
+ pfree(tmp_soundex);
+ PG_RETURN_NULL();
+ }
+
+ soundex = cstring_to_text(pg_any_to_server(tmp_soundex, strlen(tmp_soundex), PG_UTF8));
+ pfree(tmp_soundex);
+
+ PG_RETURN_TEXT_P(soundex);
+}
+
+
+typedef dm_node dm_nodes[DM_MAX_NODES];
+typedef dm_node * dm_leaves[DM_MAX_LEAVES];
+
+
+/* Template for new node in soundex code tree. */
+static const dm_node start_node = {
+ .soundex_length = 0,
+ .soundex = "000000 ", /* Six digits + joining space */
+ .is_leaf = 0,
+ .last_update = 0,
+ .code_digit = '\0',
+ .prev_code_digits = {'\0', '\0'},
+ .next_code_digits = {'\0', '\0'},
+ .prev_code_index = 0,
+ .next_code_index = 0,
+ .next_nodes = {NULL}
+};
+
+/* Dummy soundex codes at end of input. */
+static dm_codes end_codes[2] =
+{
+ {
+ "X", "X", "X"
+ }
+};
+
+
+/* Initialize soundex code tree node for next code digit. */
+static void
+initialize_node(dm_node * node, int last_update)
+{
+ if (node->last_update < last_update)
+ {
+ node->prev_code_digits[0] = node->next_code_digits[0];
+ node->prev_code_digits[1] = node->next_code_digits[1];
+ node->next_code_digits[0] = '\0';
+ node->next_code_digits[1] = '\0';
+ node->prev_code_index = node->next_code_index;
+ node->next_code_index = 0;
+ node->is_leaf = 0;
+ node->last_update = last_update;
+ }
+}
+
+
+/* Update soundex code tree node with next code digit. */
+static void
+add_next_code_digit(dm_node * node, int code_index, char code_digit)
+{
+ /* OR in index 1 or 2. */
+ node->next_code_index |= code_index;
+
+ if (!node->next_code_digits[0])
+ {
+ node->next_code_digits[0] = code_digit;
+ }
+ else if (node->next_code_digits[0] != code_digit)
+ {
+ node->next_code_digits[1] = code_digit;
+ }
+}
+
+
+/* Mark soundex code tree node as leaf. */
+static void
+set_leaf(dm_leaves leaves_next, int *num_leaves_next, dm_node * node)
+{
+ if (!node->is_leaf)
+ {
+ node->is_leaf = 1;
+ leaves_next[(*num_leaves_next)++] = node;
+ }
+}
+
+
+/* Find next node corresponding to code digit, or create a new node. */
+static dm_node * find_or_create_node(dm_nodes nodes, int *num_nodes,
+ dm_node * node, char code_digit)
+{
+ dm_node **next_nodes;
+ dm_node *next_node;
+
+ for (next_nodes = node->next_nodes; (next_node = *next_nodes); next_nodes++)
+ {
+ if (next_node->code_digit == code_digit)
+ {
+ return next_node;
+ }
+ }
+
+ next_node = &nodes[(*num_nodes)++];
+ *next_nodes = next_node;
+
+ *next_node = start_node;
+ memcpy(next_node->soundex, node->soundex, sizeof(next_node->soundex));
+ next_node->soundex_length = node->soundex_length;
+ next_node->soundex[next_node->soundex_length++] = code_digit;
+ next_node->code_digit = code_digit;
+ next_node->next_code_index = node->prev_code_index;
+
+ return next_node;
+}
+
+
+/* Update node for next code digit(s). */
+static int
+update_node(dm_nodes nodes, dm_node * node, int *num_nodes,
+ dm_leaves leaves_next, int *num_leaves_next,
+ int letter_no, int prev_code_index, int next_code_index,
+ char *next_code_digits, int digit_no)
+{
+ int i;
+ char next_code_digit = next_code_digits[digit_no];
+ int num_dirty_nodes = 0;
+ dm_node *dirty_nodes[2];
+
+ initialize_node(node, letter_no);
+
+ if (node->soundex_length == DM_MAX_CODE_DIGITS)
+ {
+ /* Keep completed soundex code. */
+ set_leaf(leaves_next, num_leaves_next, node);
+ return 0;
+ }
+
+ if (node->prev_code_index && !(node->prev_code_index & prev_code_index))
+ {
+ /*
+ * If the sound (vowel / consonant) of this letter encoding doesn't
+ * correspond to the coding index of the previous letter, we skip this
+ * letter encoding. Note that currently, only "J" can be either a
+ * vowel or a consonant.
+ */
+ return 1;
+ }
+
+ if (next_code_digit == 'X' ||
+ (digit_no == 0 &&
+ (node->prev_code_digits[0] == next_code_digit ||
+ node->prev_code_digits[1] == next_code_digit)))
+ {
+ /* The code digit is the same as one of the previous (i.e. not added). */
+ dirty_nodes[num_dirty_nodes++] = node;
+ }
+
+ if (next_code_digit != 'X' &&
+ (digit_no > 0 ||
+ node->prev_code_digits[0] != next_code_digit ||
+ node->prev_code_digits[1]))
+ {
+ /* The code digit is different from one of the previous (i.e. added). */
+ node = find_or_create_node(nodes, num_nodes, node, next_code_digit);
+ initialize_node(node, letter_no);
+ dirty_nodes[num_dirty_nodes++] = node;
+ }
+
+ for (i = 0; i < num_dirty_nodes; i++)
+ {
+ /* Add code digit leading to the current node. */
+ add_next_code_digit(dirty_nodes[i], next_code_index, next_code_digit);
+
+ if (next_code_digits[++digit_no])
+ {
+ update_node(nodes, dirty_nodes[i], num_nodes,
+ leaves_next, num_leaves_next,
+ letter_no, prev_code_index, next_code_index,
+ next_code_digits, digit_no);
+ }
+ else
+ {
+ set_leaf(leaves_next, num_leaves_next, dirty_nodes[i]);
+ }
+ }
+
+ return 1;
+}
+
+
+/* Update soundex tree leaf nodes. Return 1 when all nodes are completed. */
+static int
+update_leaves(dm_nodes nodes, int *num_nodes,
+ dm_leaves leaves[2], int *ix_leaves, int *num_leaves,
+ int letter_no, dm_codes * codes, dm_codes * next_codes)
+{
+ int i,
+ j,
+ k,
+ code_index;
+ dm_code *code,
+ *next_code;
+ int num_leaves_next = 0;
+ int ix_leaves_next = (*ix_leaves + 1) & 1; /* Alternate ix: 0, 1 */
+ int finished = 1;
+
+ for (i = 0; i < *num_leaves; i++)
+ {
+ dm_node *node = leaves[*ix_leaves][i];
+
+ /* One or two alternate code sequences. */
+ for (j = 0; j < 2 && (code = codes[j]) && code[0][0]; j++)
+ {
+ /* Coding for previous letter - before vowel: 1, all other: 2 */
+ int prev_code_index = (code[0][0] > '1') + 1;
+
+ /* One or two alternate next code sequences. */
+ for (k = 0; k < 2 && (next_code = next_codes[k]) && next_code[0][0]; k++)
+ {
+ /* Determine which code to use. */
+ if (letter_no == 0)
+ {
+ /* This is the first letter. */
+ code_index = 0;
+ }
+ else if (next_code[0][0] <= '1')
+ {
+ /* The next letter is a vowel. */
+ code_index = 1;
+ }
+ else
+ {
+ /* All other cases. */
+ code_index = 2;
+ }
+
+ /* One or two sequential code digits. */
+ if (update_node(nodes, node, num_nodes,
+ leaves[ix_leaves_next], &num_leaves_next,
+ letter_no, prev_code_index, code_index,
+ code[code_index], 0))
+ {
+ finished = 0;
+ }
+ }
+ }
+ }
+
+ *ix_leaves = ix_leaves_next;
+ *num_leaves = num_leaves_next;
+
+ return finished;
+}
+
+
+/* Mapping from ISO8859-1 to ASCII */
+static const char tr_accents_iso8859_1[] =
+/*
+"ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖ×ØÙÚÛÜÝÞßàáâãäåæçèéêëìíîïðñòóôõö÷øùúûüýþÿ"
+*/
+"AAAAAAECEEEEIIIIDNOOOOO*OUUUUYDsaaaaaaeceeeeiiiidnooooo/ouuuuydy";
+
+static char
+unaccent_iso8859_1(unsigned char c)
+{
+ return c >= 192 ? tr_accents_iso8859_1[c - 192] : c;
+}
+
+
+/* Convert an UTF-8 character to ISO-8859-1.
+ * Unconvertable characters are returned as '?'.
+ * NB! Beware of the domain specific conversion of Ą, Ę, and Ţ/Ț.
+ */
+static char
+utf8_to_iso8859_1(char *str, int *ix)
+{
+ const char unknown = '?';
+ unsigned char c,
+ c2;
+ unsigned int code_point;
+
+ /* First byte. */
+ c = (unsigned char) str[(*ix)++];
+ if (c < 0x80)
+ {
+ /* ASCII code point. */
+ if (c >= '[' && c <= ']')
+ {
+ /* Codes reserved for Ą, Ę, and Ţ/Ț. */
+ return unknown;
+ }
+
+ return c;
+ }
+
+ /* Second byte. */
+ c2 = (unsigned char) str[(*ix)++];
+ if (!c2)
+ {
+ /* The UTF-8 character is cut short (invalid code point). */
+ (*ix)--;
+ return unknown;
+ }
+
+ if (c < 0xE0)
+ {
+ /* Two-byte character. */
+ code_point = ((c & 0x1F) << 6) | (c2 & 0x3F);
+ if (code_point < 0x100)
+ {
+ /* ISO-8859-1 code point. */
+ return code_point;
+ }
+ else if (code_point == 0x0104 || code_point == 0x0105)
+ {
+ /* Ą/ą */
+ return '[';
+ }
+ else if (code_point == 0x0118 || code_point == 0x0119)
+ {
+ /* Ę/ę */
+ return '\\';
+ }
+ else if (code_point == 0x0162 || code_point == 0x0163 ||
+ code_point == 0x021A || code_point == 0x021B)
+ {
+ /* Ţ/ţ or Ț/ț */
+ return ']';
+ }
+ else
+ {
+ return unknown;
+ }
+ }
+
+ /* Third byte. */
+ if (!str[(*ix)++])
+ {
+ /* The UTF-8 character is cut short (invalid code point). */
+ (*ix)--;
+ return unknown;
+ }
+
+ if (c < 0xF0)
+ {
+ /* Three-byte character. */
+ return unknown;
+ }
+
+ /* Fourth byte. */
+ if (!str[(*ix)++])
+ {
+ /* The UTF-8 character is cut short (invalid code point). */
+ (*ix)--;
+ }
+
+ return unknown;
+}
+
+
+/* Return next character, converted from UTF-8 to uppercase ASCII. */
+static char
+read_char(char *str, int *ix)
+{
+ return toupper(unaccent_iso8859_1(utf8_to_iso8859_1(str, ix)));
+}
+
+
+/* Convert input to ASCII, skipping any characters not in [A-\]]. */
+static void
+normalize_input(char *src, char *dst)
+{
+ int c;
+ int i = 0,
+ j = 0;
+
+ while ((c = read_char(src, &i)))
+ {
+ if (c >= 'A' && c <= ']')
+ {
+ dst[j++] = c;
+ }
+ }
+
+ dst[j] = '\0';
+}
+
+
+/* Return sound coding for "letter" (letter sequence) */
+static dm_codes *
+read_letter(char *str, int *ix)
+{
+ int c,
+ cmp;
+ int i = *ix,
+ j;
+ dm_letter *letters;
+ dm_codes *codes;
+
+ /* First letter in sequence. */
+ if (!(c = str[i++]))
+ {
+ return NULL;
+ }
+ letters = &letter_[c - 'A'];
+ codes = letters->codes;
+ *ix = i;
+
+ /* Any subsequent letters in sequence. */
+ while ((letters = letters->letters) && (c = str[i++]))
+ {
+ for (j = 0; (cmp = letters[j].letter); j++)
+ {
+ if (cmp == c)
+ {
+ /* Letter found. */
+ letters = &letters[j];
+ if (letters->codes)
+ {
+ /* Coding for letter sequence found. */
+ codes = letters->codes;
+ *ix = i;
+ }
+ break;
+ }
+ }
+ if (!cmp)
+ {
+ /* The sequence of letters has no coding. */
+ break;
+ }
+ }
+
+ return codes;
+}
+
+
+/* Generate all Daitch-Mokotoff soundex codes for word, separated by space. */
+static char *
+_daitch_mokotoff(char *word, char *soundex, size_t n)
+{
+ int i = 0,
+ j;
+ int letter_no = 0;
+ int ix_leaves = 0;
+ int num_nodes = 0,
+ num_leaves = 0;
+ dm_codes *codes,
+ *next_codes;
+ dm_node *nodes;
+ dm_leaves *leaves;
+
+ /* Convert input to encodable ASCII characters, stored in soundex buffer. */
+ normalize_input(word, soundex);
+ if (!soundex[0])
+ {
+ /* No encodable character in input. */
+ return NULL;
+ }
+
+ /* Allocate memory for node tree. */
+ nodes = palloc(sizeof(dm_nodes));
+ leaves = palloc(2 * sizeof(dm_leaves));
+
+ /* Starting point. */
+ nodes[num_nodes++] = start_node;
+ leaves[ix_leaves][num_leaves++] = &nodes[0];
+
+ codes = read_letter(soundex, &i);
+
+ while (codes)
+ {
+ next_codes = read_letter(soundex, &i);
+
+ /* Update leaf nodes. */
+ if (update_leaves(nodes, &num_nodes,
+ leaves, &ix_leaves, &num_leaves,
+ letter_no, codes, next_codes ? next_codes : end_codes))
+ {
+ /* All soundex codes are completed to six digits. */
+ break;
+ }
+
+ codes = next_codes;
+ letter_no++;
+ }
+
+ /* Concatenate all generated soundex codes. */
+ for (i = 0, j = 0;
+ i < num_leaves && j + DM_MAX_CODE_DIGITS + 1 <= n;
+ i++, j += DM_MAX_CODE_DIGITS + 1)
+ {
+ memcpy(&soundex[j], leaves[ix_leaves][i]->soundex, DM_MAX_CODE_DIGITS + 1);
+ }
+
+ /* Terminate string. */
+ soundex[j - (j != 0)] = '\0';
+
+ pfree(leaves);
+ pfree(nodes);
+
+ return soundex;
+}
diff --git a/contrib/fuzzystrmatch/daitch_mokotoff.h b/contrib/fuzzystrmatch/daitch_mokotoff.h
new file mode 100644
index 0000000000..8426069825
--- /dev/null
+++ b/contrib/fuzzystrmatch/daitch_mokotoff.h
@@ -0,0 +1,999 @@
+/*
+ * Types and lookup tables for Daitch-Mokotoff Soundex
+ *
+ * Copyright (c) 2021 Finance Norway
+ * Author: Dag Lem <[email protected]>
+ *
+ * This file is generated by daitch_mokotoff_header.pl
+ *
+ * Permission to use, copy, modify, and distribute this software and its
+ * documentation for any purpose, without fee, and without a written agreement
+ * is hereby granted, provided that the above copyright notice and this
+ * paragraph and the following two paragraphs appear in all copies.
+ *
+ * IN NO EVENT SHALL THE AUTHOR OR DISTRIBUTORS BE LIABLE TO ANY PARTY FOR
+ * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING
+ * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS
+ * DOCUMENTATION, EVEN IF THE AUTHOR OR DISTRIBUTORS HAVE BEEN ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ * THE AUTHOR AND DISTRIBUTORS SPECIFICALLY DISCLAIM ANY WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
+ * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS
+ * ON AN "AS IS" BASIS, AND THE AUTHOR AND DISTRIBUTORS HAS NO OBLIGATIONS TO
+ * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
+ */
+
+#include <stdlib.h>
+
+#define DM_MAX_CODE_DIGITS 6
+#define DM_MAX_ALTERNATE_CODES 5
+#define DM_MAX_NODES 1564
+#define DM_MAX_LEAVES 1250
+#define DM_MAX_SOUNDEX_CHARS (DM_MAX_NODES*(DM_MAX_CODE_DIGITS + 1))
+
+typedef char dm_code[2 + 1]; /* One or two sequential code digits + NUL */
+typedef dm_code dm_codes[3]; /* Start of name, before a vowel, any other */
+
+/* Letter in input sequence */
+struct dm_letter
+{
+ char letter; /* Present letter in sequence */
+ struct dm_letter *letters; /* List of possible successive letters */
+ dm_codes *codes; /* Code sequence(s) for complete sequence */
+};
+
+/* Node in soundex code tree */
+struct dm_node
+{
+ int soundex_length; /* Length of generated soundex code */
+ char soundex[DM_MAX_CODE_DIGITS + 1]; /* Soundex code */
+ int is_leaf; /* Candidate for complete soundex code */
+ int last_update; /* Letter number for last update of node */
+ char code_digit; /* Last code digit, 0 - 9 */
+
+ /*
+ * One or two alternate code digits leading to this node. If there are two
+ * digits, one of them is always an 'X'. Repeated code digits and 'X' lead
+ * back to the same node.
+ */
+ char prev_code_digits[2];
+ /* One or two alternate code digits moving forward. */
+ char next_code_digits[2];
+ /* ORed together code index(es) used to reach current node. */
+ int prev_code_index;
+ int next_code_index;
+ /* Nodes branching out from this node. */
+ struct dm_node *next_nodes[DM_MAX_ALTERNATE_CODES + 1];
+};
+
+typedef struct dm_letter dm_letter;
+typedef struct dm_node dm_node;
+
+/* Codes for letter sequence at start of name, before a vowel, and any other. */
+static dm_codes codes_0_1_X[2] =
+{
+ {
+ "0", "1", "X"
+ }
+};
+static dm_codes codes_0_7_X[2] =
+{
+ {
+ "0", "7", "X"
+ }
+};
+static dm_codes codes_0_X_X[2] =
+{
+ {
+ "0", "X", "X"
+ }
+};
+static dm_codes codes_1_1_X[2] =
+{
+ {
+ "1", "1", "X"
+ }
+};
+static dm_codes codes_1_X_X[2] =
+{
+ {
+ "1", "X", "X"
+ }
+};
+static dm_codes codes_1_X_X_or_4_4_4[2] =
+{
+ {
+ "1", "X", "X"
+ },
+ {
+ "4", "4", "4"
+ }
+};
+static dm_codes codes_2_43_43[2] =
+{
+ {
+ "2", "43", "43"
+ }
+};
+static dm_codes codes_2_4_4[2] =
+{
+ {
+ "2", "4", "4"
+ }
+};
+static dm_codes codes_3_3_3[2] =
+{
+ {
+ "3", "3", "3"
+ }
+};
+static dm_codes codes_3_3_3_or_4_4_4[2] =
+{
+ {
+ "3", "3", "3"
+ },
+ {
+ "4", "4", "4"
+ }
+};
+static dm_codes codes_4_4_4[2] =
+{
+ {
+ "4", "4", "4"
+ }
+};
+static dm_codes codes_5_54_54[2] =
+{
+ {
+ "5", "54", "54"
+ }
+};
+static dm_codes codes_5_5_5[2] =
+{
+ {
+ "5", "5", "5"
+ }
+};
+static dm_codes codes_5_5_5_or_45_45_45[2] =
+{
+ {
+ "5", "5", "5"
+ },
+ {
+ "45", "45", "45"
+ }
+};
+static dm_codes codes_5_5_5_or_4_4_4[2] =
+{
+ {
+ "5", "5", "5"
+ },
+ {
+ "4", "4", "4"
+ }
+};
+static dm_codes codes_5_5_X[2] =
+{
+ {
+ "5", "5", "X"
+ }
+};
+static dm_codes codes_66_66_66[2] =
+{
+ {
+ "66", "66", "66"
+ }
+};
+static dm_codes codes_6_6_6[2] =
+{
+ {
+ "6", "6", "6"
+ }
+};
+static dm_codes codes_7_7_7[2] =
+{
+ {
+ "7", "7", "7"
+ }
+};
+static dm_codes codes_8_8_8[2] =
+{
+ {
+ "8", "8", "8"
+ }
+};
+static dm_codes codes_94_94_94_or_4_4_4[2] =
+{
+ {
+ "94", "94", "94"
+ },
+ {
+ "4", "4", "4"
+ }
+};
+static dm_codes codes_9_9_9[2] =
+{
+ {
+ "9", "9", "9"
+ }
+};
+static dm_codes codes_X_X_6_or_X_X_X[2] =
+{
+ {
+ "X", "X", "6"
+ },
+ {
+ "X", "X", "X"
+ }
+};
+
+/* Coding for alternative following letters in sequence. */
+static dm_letter letter_A[] =
+{
+ {
+ 'I', NULL, codes_0_1_X
+ },
+ {
+ 'J', NULL, codes_0_1_X
+ },
+ {
+ 'U', NULL, codes_0_7_X
+ },
+ {
+ 'Y', NULL, codes_0_1_X
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_CH[] =
+{
+ {
+ 'S', NULL, codes_5_54_54
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_CS[] =
+{
+ {
+ 'Z', NULL, codes_4_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_CZ[] =
+{
+ {
+ 'S', NULL, codes_4_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_C[] =
+{
+ {
+ 'H', letter_CH, codes_5_5_5_or_4_4_4
+ },
+ {
+ 'K', NULL, codes_5_5_5_or_45_45_45
+ },
+ {
+ 'S', letter_CS, codes_4_4_4
+ },
+ {
+ 'Z', letter_CZ, codes_4_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_DR[] =
+{
+ {
+ 'S', NULL, codes_4_4_4
+ },
+ {
+ 'Z', NULL, codes_4_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_DS[] =
+{
+ {
+ 'H', NULL, codes_4_4_4
+ },
+ {
+ 'Z', NULL, codes_4_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_DZ[] =
+{
+ {
+ 'H', NULL, codes_4_4_4
+ },
+ {
+ 'S', NULL, codes_4_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_D[] =
+{
+ {
+ 'R', letter_DR, NULL
+ },
+ {
+ 'S', letter_DS, codes_4_4_4
+ },
+ {
+ 'T', NULL, codes_3_3_3
+ },
+ {
+ 'Z', letter_DZ, codes_4_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_E[] =
+{
+ {
+ 'I', NULL, codes_0_1_X
+ },
+ {
+ 'J', NULL, codes_0_1_X
+ },
+ {
+ 'U', NULL, codes_1_1_X
+ },
+ {
+ 'Y', NULL, codes_0_1_X
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_F[] =
+{
+ {
+ 'B', NULL, codes_7_7_7
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_I[] =
+{
+ {
+ 'A', NULL, codes_1_X_X
+ },
+ {
+ 'E', NULL, codes_1_X_X
+ },
+ {
+ 'O', NULL, codes_1_X_X
+ },
+ {
+ 'U', NULL, codes_1_X_X
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_K[] =
+{
+ {
+ 'H', NULL, codes_5_5_5
+ },
+ {
+ 'S', NULL, codes_5_54_54
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_M[] =
+{
+ {
+ 'N', NULL, codes_66_66_66
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_N[] =
+{
+ {
+ 'M', NULL, codes_66_66_66
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_O[] =
+{
+ {
+ 'I', NULL, codes_0_1_X
+ },
+ {
+ 'J', NULL, codes_0_1_X
+ },
+ {
+ 'Y', NULL, codes_0_1_X
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_P[] =
+{
+ {
+ 'F', NULL, codes_7_7_7
+ },
+ {
+ 'H', NULL, codes_7_7_7
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_R[] =
+{
+ {
+ 'S', NULL, codes_94_94_94_or_4_4_4
+ },
+ {
+ 'Z', NULL, codes_94_94_94_or_4_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_SCHTC[] =
+{
+ {
+ 'H', NULL, codes_2_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_SCHTSC[] =
+{
+ {
+ 'H', NULL, codes_2_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_SCHTS[] =
+{
+ {
+ 'C', letter_SCHTSC, NULL
+ },
+ {
+ 'H', NULL, codes_2_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_SCHT[] =
+{
+ {
+ 'C', letter_SCHTC, NULL
+ },
+ {
+ 'S', letter_SCHTS, NULL
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_SCH[] =
+{
+ {
+ 'D', NULL, codes_2_43_43
+ },
+ {
+ 'T', letter_SCHT, codes_2_43_43
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_SC[] =
+{
+ {
+ 'H', letter_SCH, codes_4_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_SHC[] =
+{
+ {
+ 'H', NULL, codes_2_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_SHTC[] =
+{
+ {
+ 'H', NULL, codes_2_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_SHTS[] =
+{
+ {
+ 'H', NULL, codes_2_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_SHT[] =
+{
+ {
+ 'C', letter_SHTC, NULL
+ },
+ {
+ 'S', letter_SHTS, NULL
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_SH[] =
+{
+ {
+ 'C', letter_SHC, NULL
+ },
+ {
+ 'D', NULL, codes_2_43_43
+ },
+ {
+ 'T', letter_SHT, codes_2_43_43
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_STC[] =
+{
+ {
+ 'H', NULL, codes_2_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_STR[] =
+{
+ {
+ 'S', NULL, codes_2_4_4
+ },
+ {
+ 'Z', NULL, codes_2_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_STSC[] =
+{
+ {
+ 'H', NULL, codes_2_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_STS[] =
+{
+ {
+ 'C', letter_STSC, NULL
+ },
+ {
+ 'H', NULL, codes_2_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_ST[] =
+{
+ {
+ 'C', letter_STC, NULL
+ },
+ {
+ 'R', letter_STR, NULL
+ },
+ {
+ 'S', letter_STS, NULL
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_SZC[] =
+{
+ {
+ 'S', NULL, codes_2_4_4
+ },
+ {
+ 'Z', NULL, codes_2_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_SZ[] =
+{
+ {
+ 'C', letter_SZC, NULL
+ },
+ {
+ 'D', NULL, codes_2_43_43
+ },
+ {
+ 'T', NULL, codes_2_43_43
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_S[] =
+{
+ {
+ 'C', letter_SC, codes_2_4_4
+ },
+ {
+ 'D', NULL, codes_2_43_43
+ },
+ {
+ 'H', letter_SH, codes_4_4_4
+ },
+ {
+ 'T', letter_ST, codes_2_43_43
+ },
+ {
+ 'Z', letter_SZ, codes_4_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_TC[] =
+{
+ {
+ 'H', NULL, codes_4_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_TR[] =
+{
+ {
+ 'S', NULL, codes_4_4_4
+ },
+ {
+ 'Z', NULL, codes_4_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_TSC[] =
+{
+ {
+ 'H', NULL, codes_4_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_TS[] =
+{
+ {
+ 'C', letter_TSC, NULL
+ },
+ {
+ 'H', NULL, codes_4_4_4
+ },
+ {
+ 'Z', NULL, codes_4_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_TTC[] =
+{
+ {
+ 'H', NULL, codes_4_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_TTSC[] =
+{
+ {
+ 'H', NULL, codes_4_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_TTS[] =
+{
+ {
+ 'C', letter_TTSC, NULL
+ },
+ {
+ 'Z', NULL, codes_4_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_TT[] =
+{
+ {
+ 'C', letter_TTC, NULL
+ },
+ {
+ 'S', letter_TTS, codes_4_4_4
+ },
+ {
+ 'Z', NULL, codes_4_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_TZ[] =
+{
+ {
+ 'S', NULL, codes_4_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_T[] =
+{
+ {
+ 'C', letter_TC, codes_4_4_4
+ },
+ {
+ 'H', NULL, codes_3_3_3
+ },
+ {
+ 'R', letter_TR, NULL
+ },
+ {
+ 'S', letter_TS, codes_4_4_4
+ },
+ {
+ 'T', letter_TT, NULL
+ },
+ {
+ 'Z', letter_TZ, codes_4_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_U[] =
+{
+ {
+ 'E', NULL, codes_0_1_X
+ },
+ {
+ 'I', NULL, codes_0_1_X
+ },
+ {
+ 'J', NULL, codes_0_1_X
+ },
+ {
+ 'Y', NULL, codes_0_1_X
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_ZDZ[] =
+{
+ {
+ 'H', NULL, codes_2_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_ZD[] =
+{
+ {
+ 'Z', letter_ZDZ, codes_2_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_ZHDZ[] =
+{
+ {
+ 'H', NULL, codes_2_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_ZHD[] =
+{
+ {
+ 'Z', letter_ZHDZ, NULL
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_ZH[] =
+{
+ {
+ 'D', letter_ZHD, codes_2_43_43
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_ZSC[] =
+{
+ {
+ 'H', NULL, codes_4_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_ZS[] =
+{
+ {
+ 'C', letter_ZSC, NULL
+ },
+ {
+ 'H', NULL, codes_4_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_Z[] =
+{
+ {
+ 'D', letter_ZD, codes_2_43_43
+ },
+ {
+ 'H', letter_ZH, codes_4_4_4
+ },
+ {
+ 'S', letter_ZS, codes_4_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_[] =
+{
+ {
+ 'A', letter_A, codes_0_X_X
+ },
+ {
+ 'B', NULL, codes_7_7_7
+ },
+ {
+ 'C', letter_C, codes_5_5_5_or_4_4_4
+ },
+ {
+ 'D', letter_D, codes_3_3_3
+ },
+ {
+ 'E', letter_E, codes_0_X_X
+ },
+ {
+ 'F', letter_F, codes_7_7_7
+ },
+ {
+ 'G', NULL, codes_5_5_5
+ },
+ {
+ 'H', NULL, codes_5_5_X
+ },
+ {
+ 'I', letter_I, codes_0_X_X
+ },
+ {
+ 'J', NULL, codes_1_X_X_or_4_4_4
+ },
+ {
+ 'K', letter_K, codes_5_5_5
+ },
+ {
+ 'L', NULL, codes_8_8_8
+ },
+ {
+ 'M', letter_M, codes_6_6_6
+ },
+ {
+ 'N', letter_N, codes_6_6_6
+ },
+ {
+ 'O', letter_O, codes_0_X_X
+ },
+ {
+ 'P', letter_P, codes_7_7_7
+ },
+ {
+ 'Q', NULL, codes_5_5_5
+ },
+ {
+ 'R', letter_R, codes_9_9_9
+ },
+ {
+ 'S', letter_S, codes_4_4_4
+ },
+ {
+ 'T', letter_T, codes_3_3_3
+ },
+ {
+ 'U', letter_U, codes_0_X_X
+ },
+ {
+ 'V', NULL, codes_7_7_7
+ },
+ {
+ 'W', NULL, codes_7_7_7
+ },
+ {
+ 'X', NULL, codes_5_54_54
+ },
+ {
+ 'Y', NULL, codes_1_X_X
+ },
+ {
+ 'Z', letter_Z, codes_4_4_4
+ },
+ {
+ 'a', NULL, codes_X_X_6_or_X_X_X
+ },
+ {
+ 'e', NULL, codes_X_X_6_or_X_X_X
+ },
+ {
+ 't', NULL, codes_3_3_3_or_4_4_4
+ },
+ {
+ '\0'
+ }
+};
diff --git a/contrib/fuzzystrmatch/daitch_mokotoff_header.pl b/contrib/fuzzystrmatch/daitch_mokotoff_header.pl
new file mode 100755
index 0000000000..3e97e000ee
--- /dev/null
+++ b/contrib/fuzzystrmatch/daitch_mokotoff_header.pl
@@ -0,0 +1,288 @@
+#!/bin/perl
+#
+# Generation of types and lookup tables for Daitch-Mokotoff soundex.
+#
+# Copyright (c) 2021 Finance Norway
+# Author: Dag Lem <[email protected]>
+#
+# Permission to use, copy, modify, and distribute this software and its
+# documentation for any purpose, without fee, and without a written agreement
+# is hereby granted, provided that the above copyright notice and this
+# paragraph and the following two paragraphs appear in all copies.
+#
+# IN NO EVENT SHALL THE AUTHOR OR DISTRIBUTORS BE LIABLE TO ANY PARTY FOR
+# DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING
+# LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS
+# DOCUMENTATION, EVEN IF THE AUTHOR OR DISTRIBUTORS HAVE BEEN ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+#
+# THE AUTHOR AND DISTRIBUTORS SPECIFICALLY DISCLAIM ANY WARRANTIES,
+# INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
+# AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS
+# ON AN "AS IS" BASIS, AND THE AUTHOR AND DISTRIBUTORS HAS NO OBLIGATIONS TO
+# PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
+#
+
+use strict;
+use warnings;
+use utf8;
+use open IO => ':utf8', ':std';
+use Data::Dumper;
+
+# Parse code table and generate tree for letter transitions.
+my %codes;
+my %alternates;
+my $table = [{}, [["","",""]]];
+while (<DATA>) {
+ chomp;
+ my ($letters, $codes) = split(/\s+/);
+ my @codes = map { [ split(/,/) ] } split(/\|/, $codes);
+
+ # Find alternate code transitions for calculation of storage.
+ # The first character can never yield more than two alternate codes, and is not considered here.
+ if (@codes > 1) {
+ for my $j (1..2) {
+ for my $i (0..1) {
+ my ($a, $b) = (substr($codes[$i][$j], -1, 1), substr($codes[($i + 1)%2][$j], 0, 1));
+ $alternates{$a}{$b} = 1 if $a ne $b;
+ }
+ }
+ }
+ my $key = "codes_" . join("_or_", map { join("_", @$_) } @codes);
+ my $val = join(",\n", map { "\t{\n\t\t" . join(", ", map { "\"$_\"" } @$_) . "\n\t}" } @codes);
+ $codes{$key} = $val;
+
+ for my $letter (split(/,/, $letters)) {
+ my $ref = $table->[0];
+ # Link each character to the next in the letter combination.
+ my @c = split(//, $letter);
+ my $last_c = pop(@c);
+ for my $c (@c) {
+ $ref->{$c} //= [ {}, undef ];
+ $ref->{$c}[0] //= {};
+ $ref = $ref->{$c}[0];
+ }
+ # The sound code for the letter combination is stored at the last character.
+ $ref->{$last_c}[1] = $key;
+ }
+}
+close(DATA);
+
+# Find the maximum number of alternate codes in one position.
+my $alt_x = $alternates{"X"};
+while (my ($k, $v) = each %alternates) {
+ if (defined delete $v->{"X"}) {
+ for my $x (keys %$alt_x) {
+ $v->{$x} = 1;
+ }
+ }
+}
+my $max_alt = (reverse sort (2, map { scalar keys %$_ } values %alternates))[0];
+
+# The maximum number of nodes and leaves in the soundex tree.
+# These are safe estimates, but in practice somewhat higher than the actual maximums.
+# Note that the first character can never yield more than two alternate codes,
+# hence the calculations are performed as sums of two subtrees.
+my $digits = 6;
+# Number of nodes (sum of geometric progression).
+my $max_nodes = 2 + 2*(1 - $max_alt**($digits - 1))/(1 - $max_alt);
+# Number of leaves (exponential of base number).
+my $max_leaves = 2*$max_alt**($digits - 2);
+
+print <<EOF;
+/*
+ * Types and lookup tables for Daitch-Mokotoff Soundex
+ *
+ * Copyright (c) 2021 Finance Norway
+ * Author: Dag Lem <dag\@nimrod.no>
+ *
+ * This file is generated by daitch_mokotoff_header.pl
+ *
+ * Permission to use, copy, modify, and distribute this software and its
+ * documentation for any purpose, without fee, and without a written agreement
+ * is hereby granted, provided that the above copyright notice and this
+ * paragraph and the following two paragraphs appear in all copies.
+ *
+ * IN NO EVENT SHALL THE AUTHOR OR DISTRIBUTORS BE LIABLE TO ANY PARTY FOR
+ * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING
+ * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS
+ * DOCUMENTATION, EVEN IF THE AUTHOR OR DISTRIBUTORS HAVE BEEN ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ * THE AUTHOR AND DISTRIBUTORS SPECIFICALLY DISCLAIM ANY WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
+ * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS
+ * ON AN "AS IS" BASIS, AND THE AUTHOR AND DISTRIBUTORS HAS NO OBLIGATIONS TO
+ * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
+ */
+
+#include <stdlib.h>
+
+#define DM_MAX_CODE_DIGITS $digits
+#define DM_MAX_ALTERNATE_CODES $max_alt
+#define DM_MAX_NODES $max_nodes
+#define DM_MAX_LEAVES $max_leaves
+#define DM_MAX_SOUNDEX_CHARS (DM_MAX_NODES*(DM_MAX_CODE_DIGITS + 1))
+
+typedef char dm_code[2 + 1]; /* One or two sequential code digits + NUL */
+typedef dm_code dm_codes[3]; /* Start of name, before a vowel, any other */
+
+/* Letter in input sequence */
+struct dm_letter
+{
+ char letter; /* Present letter in sequence */
+ struct dm_letter *letters; /* List of possible successive letters */
+ dm_codes *codes; /* Code sequence(s) for complete sequence */
+};
+
+/* Node in soundex code tree */
+struct dm_node
+{
+ int soundex_length; /* Length of generated soundex code */
+ char soundex[DM_MAX_CODE_DIGITS + 1]; /* Soundex code */
+ int is_leaf; /* Candidate for complete soundex code */
+ int last_update; /* Letter number for last update of node */
+ char code_digit; /* Last code digit, 0 - 9 */
+
+ /*
+ * One or two alternate code digits leading to this node. If there are two
+ * digits, one of them is always an 'X'. Repeated code digits and 'X' lead
+ * back to the same node.
+ */
+ char prev_code_digits[2];
+ /* One or two alternate code digits moving forward. */
+ char next_code_digits[2];
+ /* ORed together code index(es) used to reach current node. */
+ int prev_code_index;
+ int next_code_index;
+ /* Nodes branching out from this node. */
+ struct dm_node *next_nodes[DM_MAX_ALTERNATE_CODES + 1];
+};
+
+typedef struct dm_letter dm_letter;
+typedef struct dm_node dm_node;
+
+/* Codes for letter sequence at start of name, before a vowel, and any other. */
+EOF
+
+for my $key (sort keys %codes) {
+ print "static dm_codes $key\[2\] =\n{\n" . $codes{$key} . "\n};\n";
+}
+
+print <<EOF;
+
+/* Coding for alternative following letters in sequence. */
+EOF
+
+sub hash2code {
+ my ($ref, $letter) = @_;
+
+ my @letters = ();
+
+ my $h = $ref->[0];
+ for my $key (sort keys %$h) {
+ $ref = $h->{$key};
+ my $children = "NULL";
+ if (defined $ref->[0]) {
+ $children = "letter_$letter$key";
+ hash2code($ref, "$letter$key");
+ }
+ my $codes = $ref->[1] // "NULL";
+ push(@letters, "\t{\n\t\t'$key', $children, $codes\n\t}");
+ }
+
+ print "static dm_letter letter_$letter\[\] =\n{\n";
+ for (@letters) {
+ print "$_,\n";
+ }
+ print "\t{\n\t\t'\\0'\n\t}\n";
+ print "};\n";
+}
+
+hash2code($table, '');
+
+
+# Table adapted from https://www.jewishgen.org/InfoFiles/Soundex.html
+#
+# X = NC (not coded)
+#
+# Note that the following letters are coded with substitute letters
+#
+# Ą => a (use '[' for table lookup)
+# Ę => e (use '\\' for table lookup)
+# Ţ => t (use ']' for table lookup)
+#
+# The rule for "UE" below does not correspond to the table referred to above,
+# however it is used by all other known implementations, including the one at
+# https://www.jewishgen.org/jos/jossound.htm (try e.g. "bouey").
+
+__DATA__
+AI,AJ,AY 0,1,X
+AU 0,7,X
+a X,X,6|X,X,X
+A 0,X,X
+B 7,7,7
+CHS 5,54,54
+CH 5,5,5|4,4,4
+CK 5,5,5|45,45,45
+CZ,CS,CSZ,CZS 4,4,4
+C 5,5,5|4,4,4
+DRZ,DRS 4,4,4
+DS,DSH,DSZ 4,4,4
+DZ,DZH,DZS 4,4,4
+D,DT 3,3,3
+EI,EJ,EY 0,1,X
+EU 1,1,X
+e X,X,6|X,X,X
+E 0,X,X
+FB 7,7,7
+F 7,7,7
+G 5,5,5
+H 5,5,X
+IA,IE,IO,IU 1,X,X
+I 0,X,X
+J 1,X,X|4,4,4
+KS 5,54,54
+KH 5,5,5
+K 5,5,5
+L 8,8,8
+MN 66,66,66
+M 6,6,6
+NM 66,66,66
+N 6,6,6
+OI,OJ,OY 0,1,X
+O 0,X,X
+P,PF,PH 7,7,7
+Q 5,5,5
+RZ,RS 94,94,94|4,4,4
+R 9,9,9
+SCHTSCH,SCHTSH,SCHTCH 2,4,4
+SCH 4,4,4
+SHTCH,SHCH,SHTSH 2,4,4
+SHT,SCHT,SCHD 2,43,43
+SH 4,4,4
+STCH,STSCH,SC 2,4,4
+STRZ,STRS,STSH 2,4,4
+ST 2,43,43
+SZCZ,SZCS 2,4,4
+SZT,SHD,SZD,SD 2,43,43
+SZ 4,4,4
+S 4,4,4
+TCH,TTCH,TTSCH 4,4,4
+TH 3,3,3
+TRZ,TRS 4,4,4
+TSCH,TSH 4,4,4
+TS,TTS,TTSZ,TC 4,4,4
+TZ,TTZ,TZS,TSZ 4,4,4
+t 3,3,3|4,4,4
+T 3,3,3
+UI,UJ,UY,UE 0,1,X
+U 0,X,X
+V 7,7,7
+W 7,7,7
+X 5,54,54
+Y 1,X,X
+ZDZ,ZDZH,ZHDZH 2,4,4
+ZD,ZHD 2,43,43
+ZH,ZS,ZSCH,ZSH 4,4,4
+Z 4,4,4
diff --git a/contrib/fuzzystrmatch/expected/fuzzystrmatch.out b/contrib/fuzzystrmatch/expected/fuzzystrmatch.out
index 493c95cdfa..f62ddad4ee 100644
--- a/contrib/fuzzystrmatch/expected/fuzzystrmatch.out
+++ b/contrib/fuzzystrmatch/expected/fuzzystrmatch.out
@@ -65,3 +65,174 @@ SELECT dmetaphone_alt('gumbo');
KMP
(1 row)
+-- Wovels
+SELECT daitch_mokotoff('Augsburg');
+ daitch_mokotoff
+-----------------
+ 054795
+(1 row)
+
+SELECT daitch_mokotoff('Breuer');
+ daitch_mokotoff
+-----------------
+ 791900
+(1 row)
+
+SELECT daitch_mokotoff('Freud');
+ daitch_mokotoff
+-----------------
+ 793000
+(1 row)
+
+-- The letter "H"
+SELECT daitch_mokotoff('Halberstadt');
+ daitch_mokotoff
+-----------------
+ 587943 587433
+(1 row)
+
+SELECT daitch_mokotoff('Mannheim');
+ daitch_mokotoff
+-----------------
+ 665600
+(1 row)
+
+-- Adjacent sounds
+SELECT daitch_mokotoff('Chernowitz');
+ daitch_mokotoff
+-----------------
+ 596740 496740
+(1 row)
+
+-- Adjacent letters with identical adjacent code digits
+SELECT daitch_mokotoff('Cherkassy');
+ daitch_mokotoff
+-----------------
+ 595400 495400
+(1 row)
+
+SELECT daitch_mokotoff('Kleinman');
+ daitch_mokotoff
+-----------------
+ 586660
+(1 row)
+
+-- More than one word
+SELECT daitch_mokotoff('Nowy Targ');
+ daitch_mokotoff
+-----------------
+ 673950
+(1 row)
+
+-- Padded with "0"
+SELECT daitch_mokotoff('Berlin');
+ daitch_mokotoff
+-----------------
+ 798600
+(1 row)
+
+-- Other examples from https://www.avotaynu.com/soundex.htm
+SELECT daitch_mokotoff('Ceniow');
+ daitch_mokotoff
+-----------------
+ 567000 467000
+(1 row)
+
+SELECT daitch_mokotoff('Tsenyuv');
+ daitch_mokotoff
+-----------------
+ 467000
+(1 row)
+
+SELECT daitch_mokotoff('Holubica');
+ daitch_mokotoff
+-----------------
+ 587500 587400
+(1 row)
+
+SELECT daitch_mokotoff('Golubitsa');
+ daitch_mokotoff
+-----------------
+ 587400
+(1 row)
+
+SELECT daitch_mokotoff('Przemysl');
+ daitch_mokotoff
+-----------------
+ 794648 746480
+(1 row)
+
+SELECT daitch_mokotoff('Pshemeshil');
+ daitch_mokotoff
+-----------------
+ 746480
+(1 row)
+
+SELECT daitch_mokotoff('Rosochowaciec');
+ daitch_mokotoff
+---------------------------------------------------------
+ 945755 945754 945745 945744 944755 944754 944745 944744
+(1 row)
+
+SELECT daitch_mokotoff('Rosokhovatsets');
+ daitch_mokotoff
+-----------------
+ 945744
+(1 row)
+
+-- Ignored characters
+SELECT daitch_mokotoff('''OBrien');
+ daitch_mokotoff
+-----------------
+ 079600
+(1 row)
+
+SELECT daitch_mokotoff('O''Brien');
+ daitch_mokotoff
+-----------------
+ 079600
+(1 row)
+
+-- "Difficult" cases, likely to cause trouble for other implementations.
+SELECT daitch_mokotoff('CJC');
+ daitch_mokotoff
+-------------------------------------------
+ 550000 540000 545000 450000 400000 440000
+(1 row)
+
+SELECT daitch_mokotoff('BESST');
+ daitch_mokotoff
+-----------------
+ 743000
+(1 row)
+
+SELECT daitch_mokotoff('BOUEY');
+ daitch_mokotoff
+-----------------
+ 710000
+(1 row)
+
+SELECT daitch_mokotoff('HANNMANN');
+ daitch_mokotoff
+-----------------
+ 566600
+(1 row)
+
+SELECT daitch_mokotoff('MCCOYJR');
+ daitch_mokotoff
+---------------------------------------------------------
+ 651900 654900 654190 654490 645190 645490 641900 644900
+(1 row)
+
+SELECT daitch_mokotoff('ACCURSO');
+ daitch_mokotoff
+---------------------------------------------------------
+ 059400 054000 054940 054400 045940 045400 049400 044000
+(1 row)
+
+SELECT daitch_mokotoff('BIERSCHBACH');
+ daitch_mokotoff
+---------------------------------------------------------
+ 794575 794574 794750 794740 745750 745740 747500 747400
+(1 row)
+
diff --git a/contrib/fuzzystrmatch/expected/fuzzystrmatch_utf8.out b/contrib/fuzzystrmatch/expected/fuzzystrmatch_utf8.out
new file mode 100644
index 0000000000..32d8260383
--- /dev/null
+++ b/contrib/fuzzystrmatch/expected/fuzzystrmatch_utf8.out
@@ -0,0 +1,61 @@
+/*
+ * This test must be run in a database with UTF-8 encoding,
+ * because other encodings don't support all the characters used.
+ */
+SELECT getdatabaseencoding() <> 'UTF8'
+ AS skip_test \gset
+\if :skip_test
+\quit
+\endif
+set client_encoding = utf8;
+-- CREATE EXTENSION IF NOT EXISTS fuzzystrmatch;
+-- Accents
+SELECT daitch_mokotoff('Müller');
+ daitch_mokotoff
+-----------------
+ 689000
+(1 row)
+
+SELECT daitch_mokotoff('Schäfer');
+ daitch_mokotoff
+-----------------
+ 479000
+(1 row)
+
+SELECT daitch_mokotoff('Straßburg');
+ daitch_mokotoff
+-----------------
+ 294795
+(1 row)
+
+SELECT daitch_mokotoff('Éregon');
+ daitch_mokotoff
+-----------------
+ 095600
+(1 row)
+
+-- Special characters added at https://www.jewishgen.org/InfoFiles/Soundex.html
+SELECT daitch_mokotoff('gąszczu');
+ daitch_mokotoff
+-----------------
+ 564000 540000
+(1 row)
+
+SELECT daitch_mokotoff('brzęczy');
+ daitch_mokotoff
+-----------------------------
+ 794640 794400 746400 744000
+(1 row)
+
+SELECT daitch_mokotoff('ţamas');
+ daitch_mokotoff
+-----------------
+ 364000 464000
+(1 row)
+
+SELECT daitch_mokotoff('țamas');
+ daitch_mokotoff
+-----------------
+ 364000 464000
+(1 row)
+
diff --git a/contrib/fuzzystrmatch/expected/fuzzystrmatch_utf8_1.out b/contrib/fuzzystrmatch/expected/fuzzystrmatch_utf8_1.out
new file mode 100644
index 0000000000..37aead89c0
--- /dev/null
+++ b/contrib/fuzzystrmatch/expected/fuzzystrmatch_utf8_1.out
@@ -0,0 +1,8 @@
+/*
+ * This test must be run in a database with UTF-8 encoding,
+ * because other encodings don't support all the characters used.
+ */
+SELECT getdatabaseencoding() <> 'UTF8'
+ AS skip_test \gset
+\if :skip_test
+\quit
diff --git a/contrib/fuzzystrmatch/fuzzystrmatch--1.1--1.2.sql b/contrib/fuzzystrmatch/fuzzystrmatch--1.1--1.2.sql
new file mode 100644
index 0000000000..b9d7b229a3
--- /dev/null
+++ b/contrib/fuzzystrmatch/fuzzystrmatch--1.1--1.2.sql
@@ -0,0 +1,8 @@
+/* contrib/fuzzystrmatch/fuzzystrmatch--1.1--1.2.sql */
+
+-- complain if script is sourced in psql, rather than via ALTER EXTENSION
+\echo Use "ALTER EXTENSION fuzzystrmatch UPDATE TO '1.2'" to load this file. \quit
+
+CREATE FUNCTION daitch_mokotoff(text) RETURNS text
+AS 'MODULE_PATHNAME', 'daitch_mokotoff'
+LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
diff --git a/contrib/fuzzystrmatch/fuzzystrmatch--1.1.sql b/contrib/fuzzystrmatch/fuzzystrmatch--1.2.sql
similarity index 92%
rename from contrib/fuzzystrmatch/fuzzystrmatch--1.1.sql
rename to contrib/fuzzystrmatch/fuzzystrmatch--1.2.sql
index 41de9d949b..2a8a100699 100644
--- a/contrib/fuzzystrmatch/fuzzystrmatch--1.1.sql
+++ b/contrib/fuzzystrmatch/fuzzystrmatch--1.2.sql
@@ -42,3 +42,7 @@ LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION dmetaphone_alt (text) RETURNS text
AS 'MODULE_PATHNAME', 'dmetaphone_alt'
LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
+
+CREATE FUNCTION daitch_mokotoff(text) RETURNS text
+AS 'MODULE_PATHNAME', 'daitch_mokotoff'
+LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
diff --git a/contrib/fuzzystrmatch/fuzzystrmatch.control b/contrib/fuzzystrmatch/fuzzystrmatch.control
index 3cd6660bf9..8b6e9fd993 100644
--- a/contrib/fuzzystrmatch/fuzzystrmatch.control
+++ b/contrib/fuzzystrmatch/fuzzystrmatch.control
@@ -1,6 +1,6 @@
# fuzzystrmatch extension
comment = 'determine similarities and distance between strings'
-default_version = '1.1'
+default_version = '1.2'
module_pathname = '$libdir/fuzzystrmatch'
relocatable = true
trusted = true
diff --git a/contrib/fuzzystrmatch/sql/fuzzystrmatch.sql b/contrib/fuzzystrmatch/sql/fuzzystrmatch.sql
index f05dc28ffb..db05c7d6b6 100644
--- a/contrib/fuzzystrmatch/sql/fuzzystrmatch.sql
+++ b/contrib/fuzzystrmatch/sql/fuzzystrmatch.sql
@@ -19,3 +19,48 @@ SELECT metaphone('GUMBO', 4);
SELECT dmetaphone('gumbo');
SELECT dmetaphone_alt('gumbo');
+
+-- Wovels
+SELECT daitch_mokotoff('Augsburg');
+SELECT daitch_mokotoff('Breuer');
+SELECT daitch_mokotoff('Freud');
+
+-- The letter "H"
+SELECT daitch_mokotoff('Halberstadt');
+SELECT daitch_mokotoff('Mannheim');
+
+-- Adjacent sounds
+SELECT daitch_mokotoff('Chernowitz');
+
+-- Adjacent letters with identical adjacent code digits
+SELECT daitch_mokotoff('Cherkassy');
+SELECT daitch_mokotoff('Kleinman');
+
+-- More than one word
+SELECT daitch_mokotoff('Nowy Targ');
+
+-- Padded with "0"
+SELECT daitch_mokotoff('Berlin');
+
+-- Other examples from https://www.avotaynu.com/soundex.htm
+SELECT daitch_mokotoff('Ceniow');
+SELECT daitch_mokotoff('Tsenyuv');
+SELECT daitch_mokotoff('Holubica');
+SELECT daitch_mokotoff('Golubitsa');
+SELECT daitch_mokotoff('Przemysl');
+SELECT daitch_mokotoff('Pshemeshil');
+SELECT daitch_mokotoff('Rosochowaciec');
+SELECT daitch_mokotoff('Rosokhovatsets');
+
+-- Ignored characters
+SELECT daitch_mokotoff('''OBrien');
+SELECT daitch_mokotoff('O''Brien');
+
+-- "Difficult" cases, likely to cause trouble for other implementations.
+SELECT daitch_mokotoff('CJC');
+SELECT daitch_mokotoff('BESST');
+SELECT daitch_mokotoff('BOUEY');
+SELECT daitch_mokotoff('HANNMANN');
+SELECT daitch_mokotoff('MCCOYJR');
+SELECT daitch_mokotoff('ACCURSO');
+SELECT daitch_mokotoff('BIERSCHBACH');
diff --git a/contrib/fuzzystrmatch/sql/fuzzystrmatch_utf8.sql b/contrib/fuzzystrmatch/sql/fuzzystrmatch_utf8.sql
new file mode 100644
index 0000000000..f42c01a1bb
--- /dev/null
+++ b/contrib/fuzzystrmatch/sql/fuzzystrmatch_utf8.sql
@@ -0,0 +1,26 @@
+/*
+ * This test must be run in a database with UTF-8 encoding,
+ * because other encodings don't support all the characters used.
+ */
+
+SELECT getdatabaseencoding() <> 'UTF8'
+ AS skip_test \gset
+\if :skip_test
+\quit
+\endif
+
+set client_encoding = utf8;
+
+-- CREATE EXTENSION IF NOT EXISTS fuzzystrmatch;
+
+-- Accents
+SELECT daitch_mokotoff('Müller');
+SELECT daitch_mokotoff('Schäfer');
+SELECT daitch_mokotoff('Straßburg');
+SELECT daitch_mokotoff('Éregon');
+
+-- Special characters added at https://www.jewishgen.org/InfoFiles/Soundex.html
+SELECT daitch_mokotoff('gąszczu');
+SELECT daitch_mokotoff('brzęczy');
+SELECT daitch_mokotoff('ţamas');
+SELECT daitch_mokotoff('țamas');
diff --git a/doc/src/sgml/fuzzystrmatch.sgml b/doc/src/sgml/fuzzystrmatch.sgml
index 382e54be91..08781778f8 100644
--- a/doc/src/sgml/fuzzystrmatch.sgml
+++ b/doc/src/sgml/fuzzystrmatch.sgml
@@ -241,4 +241,101 @@ test=# SELECT dmetaphone('gumbo');
</screen>
</sect2>
+ <sect2>
+ <title>Daitch-Mokotoff Soundex</title>
+
+ <para>
+ Compared to the American Soundex System implemented in the
+ <function>soundex</function> function, the major improvements of the
+ Daitch-Mokotoff Soundex System are:
+
+ <itemizedlist spacing="compact" mark="bullet">
+ <listitem>
+ <para>
+ Information is coded to the first six meaningful letters rather than
+ four.
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ The initial letter is coded rather than kept as is.
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ Where two consecutive letters have a single sound, they are coded as a
+ single number.
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ When a letter or combination of letters may have two different sounds,
+ it is double coded under the two different codes.
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ A letter or combination of letters maps into ten possible codes rather
+ than seven.
+ </para>
+ </listitem>
+ </itemizedlist>
+ </para>
+
+ <indexterm>
+ <primary>daitch_mokotoff</primary>
+ </indexterm>
+
+ <para>
+ The following function generates Daitch-Mokotoff soundex codes for matching
+ of similar-sounding input:
+ </para>
+
+<synopsis>
+daitch_mokotoff(text source) returns text
+</synopsis>
+
+ <para>
+ Since a Daitch-Mokotoff soundex code consists of only 6 digits,
+ <literal>source</literal> should be preferably a single word or name.
+ Any alternative soundex codes are separated by space, which makes the returned
+ text suited for use in Full Text Search, see <xref linkend="textsearch"/> and
+ <xref linkend="functions-textsearch"/>.
+ </para>
+
+ <para>
+ Example:
+ </para>
+
+<programlisting>
+CREATE OR REPLACE FUNCTION soundex_name(v_name text) RETURNS text AS $$
+ SELECT string_agg(daitch_mokotoff(n), ' ')
+ FROM regexp_split_to_table(v_name, '\s+') AS n
+$$ LANGUAGE sql STRICT IMMUTABLE PARALLEL SAFE;
+
+CREATE OR REPLACE FUNCTION soundex_tsvector(v_name text) RETURNS tsvector AS $$
+ SELECT to_tsvector('simple', soundex_name(v_name))
+$$ LANGUAGE sql STRICT IMMUTABLE PARALLEL SAFE;
+
+CREATE OR REPLACE FUNCTION soundex_tsquery(v_name text) RETURNS tsquery AS $$
+ SELECT to_tsquery('simple', quote_literal(soundex_name(v_name)))
+$$ LANGUAGE sql STRICT IMMUTABLE PARALLEL SAFE;
+
+-- Note that searches could be more efficient with the tsvector in a separate column
+-- (no recalculation on table row recheck).
+CREATE TABLE s (nm text);
+CREATE INDEX ix_s_txt ON s USING gin (soundex_tsvector(nm)) WITH (fastupdate = off);
+
+INSERT INTO s VALUES ('John Doe');
+INSERT INTO s VALUES ('Jane Roe');
+INSERT INTO s VALUES ('Public John Q.');
+INSERT INTO s VALUES ('George Best');
+
+SELECT * FROM s WHERE soundex_tsvector(nm) @@ soundex_tsquery('john');
+SELECT * FROM s WHERE soundex_tsvector(nm) @@ soundex_tsquery('jane doe');
+SELECT * FROM s WHERE soundex_tsvector(nm) @@ soundex_tsquery('john public');
+SELECT * FROM s WHERE soundex_tsvector(nm) @@ soundex_tsquery('besst, giorgio');
+</programlisting>
+ </sect2>
+
</sect1>
^ permalink raw reply [nested|flat] 19+ messages in thread
* Re: daitch_mokotoff module
2022-01-03 19:16 Re: daitch_mokotoff module Andres Freund <[email protected]>
2022-01-04 13:49 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
2022-01-04 20:18 ` Re: daitch_mokotoff module Thomas Munro <[email protected]>
2022-01-05 07:05 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
2022-01-05 20:08 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
@ 2022-02-03 14:27 ` Dag Lem <[email protected]>
2022-11-30 12:56 ` Re: daitch_mokotoff module Ian Lawrence Barwick <[email protected]>
2022-12-07 18:56 ` Re: daitch_mokotoff module Andres Freund <[email protected]>
0 siblings, 2 replies; 19+ messages in thread
From: Dag Lem @ 2022-02-03 14:27 UTC (permalink / raw)
To: PostgreSQL Hackers <[email protected]>; +Cc: Thomas Munro <[email protected]>; Julien Rouhaud <[email protected]>
Hi,
Just some minor adjustments to the patch:
* Removed call to locale-dependent toupper()
* Cleaned up input normalization
I have been asked to sign up to review a commitfest patch or patches -
unfortunately I've been ill with COVID-19 and it's not until now that
I feel well enough to have a look.
Julien: I'll have a look at https://commitfest.postgresql.org/36/3468/
as you suggested (https://commitfest.postgresql.org/36/3379/ seems to
have been reviewed now).
If there are other suggestions for a patch or patches to review for
someone new to PostgreSQL internals, I'd be grateful for that.
Best regards
Dag Lem
Attachments:
[text/x-patch] v6-daitch_mokotoff.patch (51.3K, ../../[email protected]/2-v6-daitch_mokotoff.patch)
download | inline diff:
diff --git a/contrib/fuzzystrmatch/Makefile b/contrib/fuzzystrmatch/Makefile
index 0704894f88..1d5bd84be8 100644
--- a/contrib/fuzzystrmatch/Makefile
+++ b/contrib/fuzzystrmatch/Makefile
@@ -3,14 +3,15 @@
MODULE_big = fuzzystrmatch
OBJS = \
$(WIN32RES) \
+ daitch_mokotoff.o \
dmetaphone.o \
fuzzystrmatch.o
EXTENSION = fuzzystrmatch
-DATA = fuzzystrmatch--1.1.sql fuzzystrmatch--1.0--1.1.sql
+DATA = fuzzystrmatch--1.2.sql fuzzystrmatch--1.1--1.2.sql fuzzystrmatch--1.0--1.1.sql
PGFILEDESC = "fuzzystrmatch - similarities and distance between strings"
-REGRESS = fuzzystrmatch
+REGRESS = fuzzystrmatch fuzzystrmatch_utf8
ifdef USE_PGXS
PG_CONFIG = pg_config
@@ -22,3 +23,8 @@ top_builddir = ../..
include $(top_builddir)/src/Makefile.global
include $(top_srcdir)/contrib/contrib-global.mk
endif
+
+daitch_mokotoff.o: daitch_mokotoff.h
+
+daitch_mokotoff.h: daitch_mokotoff_header.pl
+ perl $< > $@
diff --git a/contrib/fuzzystrmatch/daitch_mokotoff.c b/contrib/fuzzystrmatch/daitch_mokotoff.c
new file mode 100644
index 0000000000..ba87061845
--- /dev/null
+++ b/contrib/fuzzystrmatch/daitch_mokotoff.c
@@ -0,0 +1,587 @@
+/*
+ * Daitch-Mokotoff Soundex
+ *
+ * Copyright (c) 2021 Finance Norway
+ * Author: Dag Lem <[email protected]>
+ *
+ * This implementation of the Daitch-Mokotoff Soundex System aims at high
+ * performance.
+ *
+ * - The processing of each phoneme is initiated by an O(1) table lookup.
+ * - For phonemes containing more than one character, a coding tree is traversed
+ * to process the complete phoneme.
+ * - The (alternate) soundex codes are produced digit by digit in-place in
+ * another tree structure.
+ *
+ * References:
+ *
+ * https://www.avotaynu.com/soundex.htm
+ * https://www.jewishgen.org/InfoFiles/Soundex.html
+ * https://familypedia.fandom.com/wiki/Daitch-Mokotoff_Soundex
+ * https://stevemorse.org/census/soundex.html (dmlat.php, dmsoundex.php)
+ * https://github.com/apache/commons-codec/ (dmrules.txt, DaitchMokotoffSoundex.java)
+ * https://metacpan.org/pod/Text::Phonetic (DaitchMokotoff.pm)
+ *
+ * A few notes on other implementations:
+ *
+ * - All other known implementations have the same unofficial rules for "UE",
+ * these are also adapted by this implementation (0, 1, NC).
+ * - The only other known implementation which is capable of generating all
+ * correct soundex codes in all cases is the JOS Soundex Calculator at
+ * https://www.jewishgen.org/jos/jossound.htm
+ * - "J" is considered (only) a vowel in dmlat.php
+ * - The official rules for "RS" are commented out in dmlat.php
+ * - Identical code digits for adjacent letters are not collapsed correctly in
+ * dmsoundex.php when double digit codes are involved. E.g. "BESST" yields
+ * 744300 instead of 743000 as for "BEST".
+ * - "J" is considered (only) a consonant in DaitchMokotoffSoundex.java
+ * - "Y" is not considered a vowel in DaitchMokotoffSoundex.java
+ *
+ * Permission to use, copy, modify, and distribute this software and its
+ * documentation for any purpose, without fee, and without a written agreement
+ * is hereby granted, provided that the above copyright notice and this
+ * paragraph and the following two paragraphs appear in all copies.
+ *
+ * IN NO EVENT SHALL THE AUTHOR OR DISTRIBUTORS BE LIABLE TO ANY PARTY FOR
+ * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING
+ * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS
+ * DOCUMENTATION, EVEN IF THE AUTHOR OR DISTRIBUTORS HAVE BEEN ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ * THE AUTHOR AND DISTRIBUTORS SPECIFICALLY DISCLAIM ANY WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
+ * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS
+ * ON AN "AS IS" BASIS, AND THE AUTHOR AND DISTRIBUTORS HAS NO OBLIGATIONS TO
+ * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
+*/
+
+#include "daitch_mokotoff.h"
+
+#include "postgres.h"
+#include "utils/builtins.h"
+#include "mb/pg_wchar.h"
+
+#include <string.h>
+
+/* Internal C implementation */
+static char *_daitch_mokotoff(char *word, char *soundex, size_t n);
+
+
+PG_FUNCTION_INFO_V1(daitch_mokotoff);
+Datum
+daitch_mokotoff(PG_FUNCTION_ARGS)
+{
+ text *arg = PG_GETARG_TEXT_PP(0);
+ char *string,
+ *tmp_soundex;
+ text *soundex;
+
+ /*
+ * The maximum theoretical soundex size is several KB, however in practice
+ * anything but contrived synthetic inputs will yield a soundex size of
+ * less than 100 bytes. We thus allocate and free a temporary work buffer,
+ * and return only the actual soundex result.
+ */
+ string = pg_server_to_any(text_to_cstring(arg), VARSIZE_ANY_EXHDR(arg), PG_UTF8);
+ tmp_soundex = palloc(DM_MAX_SOUNDEX_CHARS);
+
+ if (!_daitch_mokotoff(string, tmp_soundex, DM_MAX_SOUNDEX_CHARS))
+ {
+ /* No encodable characters in input. */
+ pfree(tmp_soundex);
+ PG_RETURN_NULL();
+ }
+
+ soundex = cstring_to_text(pg_any_to_server(tmp_soundex, strlen(tmp_soundex), PG_UTF8));
+ pfree(tmp_soundex);
+
+ PG_RETURN_TEXT_P(soundex);
+}
+
+
+typedef dm_node dm_nodes[DM_MAX_NODES];
+typedef dm_node * dm_leaves[DM_MAX_LEAVES];
+
+
+/* Template for new node in soundex code tree. */
+static const dm_node start_node = {
+ .soundex_length = 0,
+ .soundex = "000000 ", /* Six digits + joining space */
+ .is_leaf = 0,
+ .last_update = 0,
+ .code_digit = '\0',
+ .prev_code_digits = {'\0', '\0'},
+ .next_code_digits = {'\0', '\0'},
+ .prev_code_index = 0,
+ .next_code_index = 0,
+ .next_nodes = {NULL}
+};
+
+/* Dummy soundex codes at end of input. */
+static dm_codes end_codes[2] =
+{
+ {
+ "X", "X", "X"
+ }
+};
+
+
+/* Initialize soundex code tree node for next code digit. */
+static void
+initialize_node(dm_node * node, int last_update)
+{
+ if (node->last_update < last_update)
+ {
+ node->prev_code_digits[0] = node->next_code_digits[0];
+ node->prev_code_digits[1] = node->next_code_digits[1];
+ node->next_code_digits[0] = '\0';
+ node->next_code_digits[1] = '\0';
+ node->prev_code_index = node->next_code_index;
+ node->next_code_index = 0;
+ node->is_leaf = 0;
+ node->last_update = last_update;
+ }
+}
+
+
+/* Update soundex code tree node with next code digit. */
+static void
+add_next_code_digit(dm_node * node, int code_index, char code_digit)
+{
+ /* OR in index 1 or 2. */
+ node->next_code_index |= code_index;
+
+ if (!node->next_code_digits[0])
+ {
+ node->next_code_digits[0] = code_digit;
+ }
+ else if (node->next_code_digits[0] != code_digit)
+ {
+ node->next_code_digits[1] = code_digit;
+ }
+}
+
+
+/* Mark soundex code tree node as leaf. */
+static void
+set_leaf(dm_leaves leaves_next, int *num_leaves_next, dm_node * node)
+{
+ if (!node->is_leaf)
+ {
+ node->is_leaf = 1;
+ leaves_next[(*num_leaves_next)++] = node;
+ }
+}
+
+
+/* Find next node corresponding to code digit, or create a new node. */
+static dm_node * find_or_create_node(dm_nodes nodes, int *num_nodes,
+ dm_node * node, char code_digit)
+{
+ dm_node **next_nodes;
+ dm_node *next_node;
+
+ for (next_nodes = node->next_nodes; (next_node = *next_nodes); next_nodes++)
+ {
+ if (next_node->code_digit == code_digit)
+ {
+ return next_node;
+ }
+ }
+
+ next_node = &nodes[(*num_nodes)++];
+ *next_nodes = next_node;
+
+ *next_node = start_node;
+ memcpy(next_node->soundex, node->soundex, sizeof(next_node->soundex));
+ next_node->soundex_length = node->soundex_length;
+ next_node->soundex[next_node->soundex_length++] = code_digit;
+ next_node->code_digit = code_digit;
+ next_node->next_code_index = node->prev_code_index;
+
+ return next_node;
+}
+
+
+/* Update node for next code digit(s). */
+static int
+update_node(dm_nodes nodes, dm_node * node, int *num_nodes,
+ dm_leaves leaves_next, int *num_leaves_next,
+ int letter_no, int prev_code_index, int next_code_index,
+ char *next_code_digits, int digit_no)
+{
+ int i;
+ char next_code_digit = next_code_digits[digit_no];
+ int num_dirty_nodes = 0;
+ dm_node *dirty_nodes[2];
+
+ initialize_node(node, letter_no);
+
+ if (node->soundex_length == DM_MAX_CODE_DIGITS)
+ {
+ /* Keep completed soundex code. */
+ set_leaf(leaves_next, num_leaves_next, node);
+ return 0;
+ }
+
+ if (node->prev_code_index && !(node->prev_code_index & prev_code_index))
+ {
+ /*
+ * If the sound (vowel / consonant) of this letter encoding doesn't
+ * correspond to the coding index of the previous letter, we skip this
+ * letter encoding. Note that currently, only "J" can be either a
+ * vowel or a consonant.
+ */
+ return 1;
+ }
+
+ if (next_code_digit == 'X' ||
+ (digit_no == 0 &&
+ (node->prev_code_digits[0] == next_code_digit ||
+ node->prev_code_digits[1] == next_code_digit)))
+ {
+ /* The code digit is the same as one of the previous (i.e. not added). */
+ dirty_nodes[num_dirty_nodes++] = node;
+ }
+
+ if (next_code_digit != 'X' &&
+ (digit_no > 0 ||
+ node->prev_code_digits[0] != next_code_digit ||
+ node->prev_code_digits[1]))
+ {
+ /* The code digit is different from one of the previous (i.e. added). */
+ node = find_or_create_node(nodes, num_nodes, node, next_code_digit);
+ initialize_node(node, letter_no);
+ dirty_nodes[num_dirty_nodes++] = node;
+ }
+
+ for (i = 0; i < num_dirty_nodes; i++)
+ {
+ /* Add code digit leading to the current node. */
+ add_next_code_digit(dirty_nodes[i], next_code_index, next_code_digit);
+
+ if (next_code_digits[++digit_no])
+ {
+ update_node(nodes, dirty_nodes[i], num_nodes,
+ leaves_next, num_leaves_next,
+ letter_no, prev_code_index, next_code_index,
+ next_code_digits, digit_no);
+ }
+ else
+ {
+ set_leaf(leaves_next, num_leaves_next, dirty_nodes[i]);
+ }
+ }
+
+ return 1;
+}
+
+
+/* Update soundex tree leaf nodes. Return 1 when all nodes are completed. */
+static int
+update_leaves(dm_nodes nodes, int *num_nodes,
+ dm_leaves leaves[2], int *ix_leaves, int *num_leaves,
+ int letter_no, dm_codes * codes, dm_codes * next_codes)
+{
+ int i,
+ j,
+ k,
+ code_index;
+ dm_code *code,
+ *next_code;
+ int num_leaves_next = 0;
+ int ix_leaves_next = (*ix_leaves + 1) & 1; /* Alternate ix: 0, 1 */
+ int finished = 1;
+
+ for (i = 0; i < *num_leaves; i++)
+ {
+ dm_node *node = leaves[*ix_leaves][i];
+
+ /* One or two alternate code sequences. */
+ for (j = 0; j < 2 && (code = codes[j]) && code[0][0]; j++)
+ {
+ /* Coding for previous letter - before vowel: 1, all other: 2 */
+ int prev_code_index = (code[0][0] > '1') + 1;
+
+ /* One or two alternate next code sequences. */
+ for (k = 0; k < 2 && (next_code = next_codes[k]) && next_code[0][0]; k++)
+ {
+ /* Determine which code to use. */
+ if (letter_no == 0)
+ {
+ /* This is the first letter. */
+ code_index = 0;
+ }
+ else if (next_code[0][0] <= '1')
+ {
+ /* The next letter is a vowel. */
+ code_index = 1;
+ }
+ else
+ {
+ /* All other cases. */
+ code_index = 2;
+ }
+
+ /* One or two sequential code digits. */
+ if (update_node(nodes, node, num_nodes,
+ leaves[ix_leaves_next], &num_leaves_next,
+ letter_no, prev_code_index, code_index,
+ code[code_index], 0))
+ {
+ finished = 0;
+ }
+ }
+ }
+ }
+
+ *ix_leaves = ix_leaves_next;
+ *num_leaves = num_leaves_next;
+
+ return finished;
+}
+
+
+/* Mapping from ISO8859-1 to upper-case ASCII */
+static const char tr_iso8859_1_to_ascii_upper[] =
+/*
+"`abcdefghijklmnopqrstuvwxyz{|}~ ¡¢£¤¥¦§¨©ª«¬ ®¯°±²³´µ¶·¸¹º»¼½¾¿ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖ×ØÙÚÛÜÝÞßàáâãäåæçèéêëìíîïðñòóôõö÷øùúûüýþÿ"
+*/
+"`ABCDEFGHIJKLMNOPQRSTUVWXYZ{|}~ ! ?AAAAAAECEEEEIIIIDNOOOOO*OUUUUYDSAAAAAAECEEEEIIIIDNOOOOO/OUUUUYDY";
+
+static char
+iso8859_1_to_ascii_upper(unsigned char c)
+{
+ return c >= 0x60 ? tr_iso8859_1_to_ascii_upper[c - 0x60] : c;
+}
+
+
+/* Convert an UTF-8 character to ISO-8859-1.
+ * Unconvertable characters are returned as '?'.
+ * NB! Beware of the domain specific conversion of Ą, Ę, and Ţ/Ț.
+ */
+static char
+utf8_to_iso8859_1(char *str, int *ix)
+{
+ const char unknown = '?';
+ unsigned char c,
+ c2;
+ unsigned int code_point;
+
+ /* First byte. */
+ c = (unsigned char) str[(*ix)++];
+ if (c < 0x80)
+ {
+ /* ASCII code point. */
+ if (c >= '[' && c <= ']')
+ {
+ /* Codes reserved for Ą, Ę, and Ţ/Ț. */
+ return unknown;
+ }
+
+ return c;
+ }
+
+ /* Second byte. */
+ c2 = (unsigned char) str[(*ix)++];
+ if (!c2)
+ {
+ /* The UTF-8 character is cut short (invalid code point). */
+ (*ix)--;
+ return unknown;
+ }
+
+ if (c < 0xE0)
+ {
+ /* Two-byte character. */
+ code_point = ((c & 0x1F) << 6) | (c2 & 0x3F);
+ if (code_point < 0x100)
+ {
+ /* ISO-8859-1 code point. */
+ return code_point;
+ }
+ else if (code_point == 0x0104 || code_point == 0x0105)
+ {
+ /* Ą/ą */
+ return '[';
+ }
+ else if (code_point == 0x0118 || code_point == 0x0119)
+ {
+ /* Ę/ę */
+ return '\\';
+ }
+ else if (code_point == 0x0162 || code_point == 0x0163 ||
+ code_point == 0x021A || code_point == 0x021B)
+ {
+ /* Ţ/ţ or Ț/ț */
+ return ']';
+ }
+ else
+ {
+ return unknown;
+ }
+ }
+
+ /* Third byte. */
+ if (!str[(*ix)++])
+ {
+ /* The UTF-8 character is cut short (invalid code point). */
+ (*ix)--;
+ return unknown;
+ }
+
+ if (c < 0xF0)
+ {
+ /* Three-byte character. */
+ return unknown;
+ }
+
+ /* Fourth byte. */
+ if (!str[(*ix)++])
+ {
+ /* The UTF-8 character is cut short (invalid code point). */
+ (*ix)--;
+ }
+
+ return unknown;
+}
+
+
+/* Return next character, converted from UTF-8 to uppercase ASCII. */
+static char
+read_char(char *str, int *ix)
+{
+ return iso8859_1_to_ascii_upper(utf8_to_iso8859_1(str, ix));
+}
+
+
+/* Read next ASCII character, skipping any characters not in [A-\]]. */
+static char
+read_valid_char(char *str, int *ix)
+{
+ char c;
+
+ while ((c = read_char(str, ix)))
+ {
+ if (c >= 'A' && c <= ']')
+ {
+ break;
+ }
+ }
+
+ return c;
+}
+
+
+/* Return sound coding for "letter" (letter sequence) */
+static dm_codes *
+read_letter(char *str, int *ix)
+{
+ char c,
+ cmp;
+ int i,
+ j;
+ dm_letter *letters;
+ dm_codes *codes;
+
+ /* First letter in sequence. */
+ if (!(c = read_valid_char(str, ix)))
+ {
+ return NULL;
+ }
+ letters = &letter_[c - 'A'];
+ codes = letters->codes;
+ i = *ix;
+
+ /* Any subsequent letters in sequence. */
+ while ((letters = letters->letters) && (c = read_valid_char(str, &i)))
+ {
+ for (j = 0; (cmp = letters[j].letter); j++)
+ {
+ if (cmp == c)
+ {
+ /* Letter found. */
+ letters = &letters[j];
+ if (letters->codes)
+ {
+ /* Coding for letter sequence found. */
+ codes = letters->codes;
+ *ix = i;
+ }
+ break;
+ }
+ }
+ if (!cmp)
+ {
+ /* The sequence of letters has no coding. */
+ break;
+ }
+ }
+
+ return codes;
+}
+
+
+/* Generate all Daitch-Mokotoff soundex codes for word, separated by space. */
+static char *
+_daitch_mokotoff(char *word, char *soundex, size_t n)
+{
+ int i = 0,
+ j;
+ int letter_no = 0;
+ int ix_leaves = 0;
+ int num_nodes = 0,
+ num_leaves = 0;
+ dm_codes *codes,
+ *next_codes;
+ dm_node *nodes;
+ dm_leaves *leaves;
+
+ /* First letter. */
+ if (!(codes = read_letter(word, &i)))
+ {
+ /* No encodable character in input. */
+ return NULL;
+ }
+
+ /* Allocate memory for node tree. */
+ nodes = palloc(sizeof(dm_nodes));
+ leaves = palloc(2 * sizeof(dm_leaves));
+
+ /* Starting point. */
+ nodes[num_nodes++] = start_node;
+ leaves[ix_leaves][num_leaves++] = &nodes[0];
+
+ while (codes)
+ {
+ next_codes = read_letter(word, &i);
+
+ /* Update leaf nodes. */
+ if (update_leaves(nodes, &num_nodes,
+ leaves, &ix_leaves, &num_leaves,
+ letter_no, codes, next_codes ? next_codes : end_codes))
+ {
+ /* All soundex codes are completed to six digits. */
+ break;
+ }
+
+ codes = next_codes;
+ letter_no++;
+ }
+
+ /* Concatenate all generated soundex codes. */
+ for (i = 0, j = 0;
+ i < num_leaves && j + DM_MAX_CODE_DIGITS + 1 <= n;
+ i++, j += DM_MAX_CODE_DIGITS + 1)
+ {
+ memcpy(&soundex[j], leaves[ix_leaves][i]->soundex, DM_MAX_CODE_DIGITS + 1);
+ }
+
+ /* Terminate string. */
+ soundex[j - 1] = '\0';
+
+ pfree(leaves);
+ pfree(nodes);
+
+ return soundex;
+}
diff --git a/contrib/fuzzystrmatch/daitch_mokotoff.h b/contrib/fuzzystrmatch/daitch_mokotoff.h
new file mode 100644
index 0000000000..8426069825
--- /dev/null
+++ b/contrib/fuzzystrmatch/daitch_mokotoff.h
@@ -0,0 +1,999 @@
+/*
+ * Types and lookup tables for Daitch-Mokotoff Soundex
+ *
+ * Copyright (c) 2021 Finance Norway
+ * Author: Dag Lem <[email protected]>
+ *
+ * This file is generated by daitch_mokotoff_header.pl
+ *
+ * Permission to use, copy, modify, and distribute this software and its
+ * documentation for any purpose, without fee, and without a written agreement
+ * is hereby granted, provided that the above copyright notice and this
+ * paragraph and the following two paragraphs appear in all copies.
+ *
+ * IN NO EVENT SHALL THE AUTHOR OR DISTRIBUTORS BE LIABLE TO ANY PARTY FOR
+ * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING
+ * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS
+ * DOCUMENTATION, EVEN IF THE AUTHOR OR DISTRIBUTORS HAVE BEEN ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ * THE AUTHOR AND DISTRIBUTORS SPECIFICALLY DISCLAIM ANY WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
+ * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS
+ * ON AN "AS IS" BASIS, AND THE AUTHOR AND DISTRIBUTORS HAS NO OBLIGATIONS TO
+ * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
+ */
+
+#include <stdlib.h>
+
+#define DM_MAX_CODE_DIGITS 6
+#define DM_MAX_ALTERNATE_CODES 5
+#define DM_MAX_NODES 1564
+#define DM_MAX_LEAVES 1250
+#define DM_MAX_SOUNDEX_CHARS (DM_MAX_NODES*(DM_MAX_CODE_DIGITS + 1))
+
+typedef char dm_code[2 + 1]; /* One or two sequential code digits + NUL */
+typedef dm_code dm_codes[3]; /* Start of name, before a vowel, any other */
+
+/* Letter in input sequence */
+struct dm_letter
+{
+ char letter; /* Present letter in sequence */
+ struct dm_letter *letters; /* List of possible successive letters */
+ dm_codes *codes; /* Code sequence(s) for complete sequence */
+};
+
+/* Node in soundex code tree */
+struct dm_node
+{
+ int soundex_length; /* Length of generated soundex code */
+ char soundex[DM_MAX_CODE_DIGITS + 1]; /* Soundex code */
+ int is_leaf; /* Candidate for complete soundex code */
+ int last_update; /* Letter number for last update of node */
+ char code_digit; /* Last code digit, 0 - 9 */
+
+ /*
+ * One or two alternate code digits leading to this node. If there are two
+ * digits, one of them is always an 'X'. Repeated code digits and 'X' lead
+ * back to the same node.
+ */
+ char prev_code_digits[2];
+ /* One or two alternate code digits moving forward. */
+ char next_code_digits[2];
+ /* ORed together code index(es) used to reach current node. */
+ int prev_code_index;
+ int next_code_index;
+ /* Nodes branching out from this node. */
+ struct dm_node *next_nodes[DM_MAX_ALTERNATE_CODES + 1];
+};
+
+typedef struct dm_letter dm_letter;
+typedef struct dm_node dm_node;
+
+/* Codes for letter sequence at start of name, before a vowel, and any other. */
+static dm_codes codes_0_1_X[2] =
+{
+ {
+ "0", "1", "X"
+ }
+};
+static dm_codes codes_0_7_X[2] =
+{
+ {
+ "0", "7", "X"
+ }
+};
+static dm_codes codes_0_X_X[2] =
+{
+ {
+ "0", "X", "X"
+ }
+};
+static dm_codes codes_1_1_X[2] =
+{
+ {
+ "1", "1", "X"
+ }
+};
+static dm_codes codes_1_X_X[2] =
+{
+ {
+ "1", "X", "X"
+ }
+};
+static dm_codes codes_1_X_X_or_4_4_4[2] =
+{
+ {
+ "1", "X", "X"
+ },
+ {
+ "4", "4", "4"
+ }
+};
+static dm_codes codes_2_43_43[2] =
+{
+ {
+ "2", "43", "43"
+ }
+};
+static dm_codes codes_2_4_4[2] =
+{
+ {
+ "2", "4", "4"
+ }
+};
+static dm_codes codes_3_3_3[2] =
+{
+ {
+ "3", "3", "3"
+ }
+};
+static dm_codes codes_3_3_3_or_4_4_4[2] =
+{
+ {
+ "3", "3", "3"
+ },
+ {
+ "4", "4", "4"
+ }
+};
+static dm_codes codes_4_4_4[2] =
+{
+ {
+ "4", "4", "4"
+ }
+};
+static dm_codes codes_5_54_54[2] =
+{
+ {
+ "5", "54", "54"
+ }
+};
+static dm_codes codes_5_5_5[2] =
+{
+ {
+ "5", "5", "5"
+ }
+};
+static dm_codes codes_5_5_5_or_45_45_45[2] =
+{
+ {
+ "5", "5", "5"
+ },
+ {
+ "45", "45", "45"
+ }
+};
+static dm_codes codes_5_5_5_or_4_4_4[2] =
+{
+ {
+ "5", "5", "5"
+ },
+ {
+ "4", "4", "4"
+ }
+};
+static dm_codes codes_5_5_X[2] =
+{
+ {
+ "5", "5", "X"
+ }
+};
+static dm_codes codes_66_66_66[2] =
+{
+ {
+ "66", "66", "66"
+ }
+};
+static dm_codes codes_6_6_6[2] =
+{
+ {
+ "6", "6", "6"
+ }
+};
+static dm_codes codes_7_7_7[2] =
+{
+ {
+ "7", "7", "7"
+ }
+};
+static dm_codes codes_8_8_8[2] =
+{
+ {
+ "8", "8", "8"
+ }
+};
+static dm_codes codes_94_94_94_or_4_4_4[2] =
+{
+ {
+ "94", "94", "94"
+ },
+ {
+ "4", "4", "4"
+ }
+};
+static dm_codes codes_9_9_9[2] =
+{
+ {
+ "9", "9", "9"
+ }
+};
+static dm_codes codes_X_X_6_or_X_X_X[2] =
+{
+ {
+ "X", "X", "6"
+ },
+ {
+ "X", "X", "X"
+ }
+};
+
+/* Coding for alternative following letters in sequence. */
+static dm_letter letter_A[] =
+{
+ {
+ 'I', NULL, codes_0_1_X
+ },
+ {
+ 'J', NULL, codes_0_1_X
+ },
+ {
+ 'U', NULL, codes_0_7_X
+ },
+ {
+ 'Y', NULL, codes_0_1_X
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_CH[] =
+{
+ {
+ 'S', NULL, codes_5_54_54
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_CS[] =
+{
+ {
+ 'Z', NULL, codes_4_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_CZ[] =
+{
+ {
+ 'S', NULL, codes_4_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_C[] =
+{
+ {
+ 'H', letter_CH, codes_5_5_5_or_4_4_4
+ },
+ {
+ 'K', NULL, codes_5_5_5_or_45_45_45
+ },
+ {
+ 'S', letter_CS, codes_4_4_4
+ },
+ {
+ 'Z', letter_CZ, codes_4_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_DR[] =
+{
+ {
+ 'S', NULL, codes_4_4_4
+ },
+ {
+ 'Z', NULL, codes_4_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_DS[] =
+{
+ {
+ 'H', NULL, codes_4_4_4
+ },
+ {
+ 'Z', NULL, codes_4_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_DZ[] =
+{
+ {
+ 'H', NULL, codes_4_4_4
+ },
+ {
+ 'S', NULL, codes_4_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_D[] =
+{
+ {
+ 'R', letter_DR, NULL
+ },
+ {
+ 'S', letter_DS, codes_4_4_4
+ },
+ {
+ 'T', NULL, codes_3_3_3
+ },
+ {
+ 'Z', letter_DZ, codes_4_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_E[] =
+{
+ {
+ 'I', NULL, codes_0_1_X
+ },
+ {
+ 'J', NULL, codes_0_1_X
+ },
+ {
+ 'U', NULL, codes_1_1_X
+ },
+ {
+ 'Y', NULL, codes_0_1_X
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_F[] =
+{
+ {
+ 'B', NULL, codes_7_7_7
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_I[] =
+{
+ {
+ 'A', NULL, codes_1_X_X
+ },
+ {
+ 'E', NULL, codes_1_X_X
+ },
+ {
+ 'O', NULL, codes_1_X_X
+ },
+ {
+ 'U', NULL, codes_1_X_X
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_K[] =
+{
+ {
+ 'H', NULL, codes_5_5_5
+ },
+ {
+ 'S', NULL, codes_5_54_54
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_M[] =
+{
+ {
+ 'N', NULL, codes_66_66_66
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_N[] =
+{
+ {
+ 'M', NULL, codes_66_66_66
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_O[] =
+{
+ {
+ 'I', NULL, codes_0_1_X
+ },
+ {
+ 'J', NULL, codes_0_1_X
+ },
+ {
+ 'Y', NULL, codes_0_1_X
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_P[] =
+{
+ {
+ 'F', NULL, codes_7_7_7
+ },
+ {
+ 'H', NULL, codes_7_7_7
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_R[] =
+{
+ {
+ 'S', NULL, codes_94_94_94_or_4_4_4
+ },
+ {
+ 'Z', NULL, codes_94_94_94_or_4_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_SCHTC[] =
+{
+ {
+ 'H', NULL, codes_2_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_SCHTSC[] =
+{
+ {
+ 'H', NULL, codes_2_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_SCHTS[] =
+{
+ {
+ 'C', letter_SCHTSC, NULL
+ },
+ {
+ 'H', NULL, codes_2_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_SCHT[] =
+{
+ {
+ 'C', letter_SCHTC, NULL
+ },
+ {
+ 'S', letter_SCHTS, NULL
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_SCH[] =
+{
+ {
+ 'D', NULL, codes_2_43_43
+ },
+ {
+ 'T', letter_SCHT, codes_2_43_43
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_SC[] =
+{
+ {
+ 'H', letter_SCH, codes_4_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_SHC[] =
+{
+ {
+ 'H', NULL, codes_2_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_SHTC[] =
+{
+ {
+ 'H', NULL, codes_2_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_SHTS[] =
+{
+ {
+ 'H', NULL, codes_2_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_SHT[] =
+{
+ {
+ 'C', letter_SHTC, NULL
+ },
+ {
+ 'S', letter_SHTS, NULL
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_SH[] =
+{
+ {
+ 'C', letter_SHC, NULL
+ },
+ {
+ 'D', NULL, codes_2_43_43
+ },
+ {
+ 'T', letter_SHT, codes_2_43_43
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_STC[] =
+{
+ {
+ 'H', NULL, codes_2_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_STR[] =
+{
+ {
+ 'S', NULL, codes_2_4_4
+ },
+ {
+ 'Z', NULL, codes_2_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_STSC[] =
+{
+ {
+ 'H', NULL, codes_2_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_STS[] =
+{
+ {
+ 'C', letter_STSC, NULL
+ },
+ {
+ 'H', NULL, codes_2_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_ST[] =
+{
+ {
+ 'C', letter_STC, NULL
+ },
+ {
+ 'R', letter_STR, NULL
+ },
+ {
+ 'S', letter_STS, NULL
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_SZC[] =
+{
+ {
+ 'S', NULL, codes_2_4_4
+ },
+ {
+ 'Z', NULL, codes_2_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_SZ[] =
+{
+ {
+ 'C', letter_SZC, NULL
+ },
+ {
+ 'D', NULL, codes_2_43_43
+ },
+ {
+ 'T', NULL, codes_2_43_43
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_S[] =
+{
+ {
+ 'C', letter_SC, codes_2_4_4
+ },
+ {
+ 'D', NULL, codes_2_43_43
+ },
+ {
+ 'H', letter_SH, codes_4_4_4
+ },
+ {
+ 'T', letter_ST, codes_2_43_43
+ },
+ {
+ 'Z', letter_SZ, codes_4_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_TC[] =
+{
+ {
+ 'H', NULL, codes_4_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_TR[] =
+{
+ {
+ 'S', NULL, codes_4_4_4
+ },
+ {
+ 'Z', NULL, codes_4_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_TSC[] =
+{
+ {
+ 'H', NULL, codes_4_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_TS[] =
+{
+ {
+ 'C', letter_TSC, NULL
+ },
+ {
+ 'H', NULL, codes_4_4_4
+ },
+ {
+ 'Z', NULL, codes_4_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_TTC[] =
+{
+ {
+ 'H', NULL, codes_4_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_TTSC[] =
+{
+ {
+ 'H', NULL, codes_4_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_TTS[] =
+{
+ {
+ 'C', letter_TTSC, NULL
+ },
+ {
+ 'Z', NULL, codes_4_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_TT[] =
+{
+ {
+ 'C', letter_TTC, NULL
+ },
+ {
+ 'S', letter_TTS, codes_4_4_4
+ },
+ {
+ 'Z', NULL, codes_4_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_TZ[] =
+{
+ {
+ 'S', NULL, codes_4_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_T[] =
+{
+ {
+ 'C', letter_TC, codes_4_4_4
+ },
+ {
+ 'H', NULL, codes_3_3_3
+ },
+ {
+ 'R', letter_TR, NULL
+ },
+ {
+ 'S', letter_TS, codes_4_4_4
+ },
+ {
+ 'T', letter_TT, NULL
+ },
+ {
+ 'Z', letter_TZ, codes_4_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_U[] =
+{
+ {
+ 'E', NULL, codes_0_1_X
+ },
+ {
+ 'I', NULL, codes_0_1_X
+ },
+ {
+ 'J', NULL, codes_0_1_X
+ },
+ {
+ 'Y', NULL, codes_0_1_X
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_ZDZ[] =
+{
+ {
+ 'H', NULL, codes_2_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_ZD[] =
+{
+ {
+ 'Z', letter_ZDZ, codes_2_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_ZHDZ[] =
+{
+ {
+ 'H', NULL, codes_2_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_ZHD[] =
+{
+ {
+ 'Z', letter_ZHDZ, NULL
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_ZH[] =
+{
+ {
+ 'D', letter_ZHD, codes_2_43_43
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_ZSC[] =
+{
+ {
+ 'H', NULL, codes_4_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_ZS[] =
+{
+ {
+ 'C', letter_ZSC, NULL
+ },
+ {
+ 'H', NULL, codes_4_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_Z[] =
+{
+ {
+ 'D', letter_ZD, codes_2_43_43
+ },
+ {
+ 'H', letter_ZH, codes_4_4_4
+ },
+ {
+ 'S', letter_ZS, codes_4_4_4
+ },
+ {
+ '\0'
+ }
+};
+static dm_letter letter_[] =
+{
+ {
+ 'A', letter_A, codes_0_X_X
+ },
+ {
+ 'B', NULL, codes_7_7_7
+ },
+ {
+ 'C', letter_C, codes_5_5_5_or_4_4_4
+ },
+ {
+ 'D', letter_D, codes_3_3_3
+ },
+ {
+ 'E', letter_E, codes_0_X_X
+ },
+ {
+ 'F', letter_F, codes_7_7_7
+ },
+ {
+ 'G', NULL, codes_5_5_5
+ },
+ {
+ 'H', NULL, codes_5_5_X
+ },
+ {
+ 'I', letter_I, codes_0_X_X
+ },
+ {
+ 'J', NULL, codes_1_X_X_or_4_4_4
+ },
+ {
+ 'K', letter_K, codes_5_5_5
+ },
+ {
+ 'L', NULL, codes_8_8_8
+ },
+ {
+ 'M', letter_M, codes_6_6_6
+ },
+ {
+ 'N', letter_N, codes_6_6_6
+ },
+ {
+ 'O', letter_O, codes_0_X_X
+ },
+ {
+ 'P', letter_P, codes_7_7_7
+ },
+ {
+ 'Q', NULL, codes_5_5_5
+ },
+ {
+ 'R', letter_R, codes_9_9_9
+ },
+ {
+ 'S', letter_S, codes_4_4_4
+ },
+ {
+ 'T', letter_T, codes_3_3_3
+ },
+ {
+ 'U', letter_U, codes_0_X_X
+ },
+ {
+ 'V', NULL, codes_7_7_7
+ },
+ {
+ 'W', NULL, codes_7_7_7
+ },
+ {
+ 'X', NULL, codes_5_54_54
+ },
+ {
+ 'Y', NULL, codes_1_X_X
+ },
+ {
+ 'Z', letter_Z, codes_4_4_4
+ },
+ {
+ 'a', NULL, codes_X_X_6_or_X_X_X
+ },
+ {
+ 'e', NULL, codes_X_X_6_or_X_X_X
+ },
+ {
+ 't', NULL, codes_3_3_3_or_4_4_4
+ },
+ {
+ '\0'
+ }
+};
diff --git a/contrib/fuzzystrmatch/daitch_mokotoff_header.pl b/contrib/fuzzystrmatch/daitch_mokotoff_header.pl
new file mode 100755
index 0000000000..3e97e000ee
--- /dev/null
+++ b/contrib/fuzzystrmatch/daitch_mokotoff_header.pl
@@ -0,0 +1,288 @@
+#!/bin/perl
+#
+# Generation of types and lookup tables for Daitch-Mokotoff soundex.
+#
+# Copyright (c) 2021 Finance Norway
+# Author: Dag Lem <[email protected]>
+#
+# Permission to use, copy, modify, and distribute this software and its
+# documentation for any purpose, without fee, and without a written agreement
+# is hereby granted, provided that the above copyright notice and this
+# paragraph and the following two paragraphs appear in all copies.
+#
+# IN NO EVENT SHALL THE AUTHOR OR DISTRIBUTORS BE LIABLE TO ANY PARTY FOR
+# DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING
+# LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS
+# DOCUMENTATION, EVEN IF THE AUTHOR OR DISTRIBUTORS HAVE BEEN ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+#
+# THE AUTHOR AND DISTRIBUTORS SPECIFICALLY DISCLAIM ANY WARRANTIES,
+# INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
+# AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS
+# ON AN "AS IS" BASIS, AND THE AUTHOR AND DISTRIBUTORS HAS NO OBLIGATIONS TO
+# PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
+#
+
+use strict;
+use warnings;
+use utf8;
+use open IO => ':utf8', ':std';
+use Data::Dumper;
+
+# Parse code table and generate tree for letter transitions.
+my %codes;
+my %alternates;
+my $table = [{}, [["","",""]]];
+while (<DATA>) {
+ chomp;
+ my ($letters, $codes) = split(/\s+/);
+ my @codes = map { [ split(/,/) ] } split(/\|/, $codes);
+
+ # Find alternate code transitions for calculation of storage.
+ # The first character can never yield more than two alternate codes, and is not considered here.
+ if (@codes > 1) {
+ for my $j (1..2) {
+ for my $i (0..1) {
+ my ($a, $b) = (substr($codes[$i][$j], -1, 1), substr($codes[($i + 1)%2][$j], 0, 1));
+ $alternates{$a}{$b} = 1 if $a ne $b;
+ }
+ }
+ }
+ my $key = "codes_" . join("_or_", map { join("_", @$_) } @codes);
+ my $val = join(",\n", map { "\t{\n\t\t" . join(", ", map { "\"$_\"" } @$_) . "\n\t}" } @codes);
+ $codes{$key} = $val;
+
+ for my $letter (split(/,/, $letters)) {
+ my $ref = $table->[0];
+ # Link each character to the next in the letter combination.
+ my @c = split(//, $letter);
+ my $last_c = pop(@c);
+ for my $c (@c) {
+ $ref->{$c} //= [ {}, undef ];
+ $ref->{$c}[0] //= {};
+ $ref = $ref->{$c}[0];
+ }
+ # The sound code for the letter combination is stored at the last character.
+ $ref->{$last_c}[1] = $key;
+ }
+}
+close(DATA);
+
+# Find the maximum number of alternate codes in one position.
+my $alt_x = $alternates{"X"};
+while (my ($k, $v) = each %alternates) {
+ if (defined delete $v->{"X"}) {
+ for my $x (keys %$alt_x) {
+ $v->{$x} = 1;
+ }
+ }
+}
+my $max_alt = (reverse sort (2, map { scalar keys %$_ } values %alternates))[0];
+
+# The maximum number of nodes and leaves in the soundex tree.
+# These are safe estimates, but in practice somewhat higher than the actual maximums.
+# Note that the first character can never yield more than two alternate codes,
+# hence the calculations are performed as sums of two subtrees.
+my $digits = 6;
+# Number of nodes (sum of geometric progression).
+my $max_nodes = 2 + 2*(1 - $max_alt**($digits - 1))/(1 - $max_alt);
+# Number of leaves (exponential of base number).
+my $max_leaves = 2*$max_alt**($digits - 2);
+
+print <<EOF;
+/*
+ * Types and lookup tables for Daitch-Mokotoff Soundex
+ *
+ * Copyright (c) 2021 Finance Norway
+ * Author: Dag Lem <dag\@nimrod.no>
+ *
+ * This file is generated by daitch_mokotoff_header.pl
+ *
+ * Permission to use, copy, modify, and distribute this software and its
+ * documentation for any purpose, without fee, and without a written agreement
+ * is hereby granted, provided that the above copyright notice and this
+ * paragraph and the following two paragraphs appear in all copies.
+ *
+ * IN NO EVENT SHALL THE AUTHOR OR DISTRIBUTORS BE LIABLE TO ANY PARTY FOR
+ * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING
+ * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS
+ * DOCUMENTATION, EVEN IF THE AUTHOR OR DISTRIBUTORS HAVE BEEN ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ * THE AUTHOR AND DISTRIBUTORS SPECIFICALLY DISCLAIM ANY WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
+ * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS
+ * ON AN "AS IS" BASIS, AND THE AUTHOR AND DISTRIBUTORS HAS NO OBLIGATIONS TO
+ * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
+ */
+
+#include <stdlib.h>
+
+#define DM_MAX_CODE_DIGITS $digits
+#define DM_MAX_ALTERNATE_CODES $max_alt
+#define DM_MAX_NODES $max_nodes
+#define DM_MAX_LEAVES $max_leaves
+#define DM_MAX_SOUNDEX_CHARS (DM_MAX_NODES*(DM_MAX_CODE_DIGITS + 1))
+
+typedef char dm_code[2 + 1]; /* One or two sequential code digits + NUL */
+typedef dm_code dm_codes[3]; /* Start of name, before a vowel, any other */
+
+/* Letter in input sequence */
+struct dm_letter
+{
+ char letter; /* Present letter in sequence */
+ struct dm_letter *letters; /* List of possible successive letters */
+ dm_codes *codes; /* Code sequence(s) for complete sequence */
+};
+
+/* Node in soundex code tree */
+struct dm_node
+{
+ int soundex_length; /* Length of generated soundex code */
+ char soundex[DM_MAX_CODE_DIGITS + 1]; /* Soundex code */
+ int is_leaf; /* Candidate for complete soundex code */
+ int last_update; /* Letter number for last update of node */
+ char code_digit; /* Last code digit, 0 - 9 */
+
+ /*
+ * One or two alternate code digits leading to this node. If there are two
+ * digits, one of them is always an 'X'. Repeated code digits and 'X' lead
+ * back to the same node.
+ */
+ char prev_code_digits[2];
+ /* One or two alternate code digits moving forward. */
+ char next_code_digits[2];
+ /* ORed together code index(es) used to reach current node. */
+ int prev_code_index;
+ int next_code_index;
+ /* Nodes branching out from this node. */
+ struct dm_node *next_nodes[DM_MAX_ALTERNATE_CODES + 1];
+};
+
+typedef struct dm_letter dm_letter;
+typedef struct dm_node dm_node;
+
+/* Codes for letter sequence at start of name, before a vowel, and any other. */
+EOF
+
+for my $key (sort keys %codes) {
+ print "static dm_codes $key\[2\] =\n{\n" . $codes{$key} . "\n};\n";
+}
+
+print <<EOF;
+
+/* Coding for alternative following letters in sequence. */
+EOF
+
+sub hash2code {
+ my ($ref, $letter) = @_;
+
+ my @letters = ();
+
+ my $h = $ref->[0];
+ for my $key (sort keys %$h) {
+ $ref = $h->{$key};
+ my $children = "NULL";
+ if (defined $ref->[0]) {
+ $children = "letter_$letter$key";
+ hash2code($ref, "$letter$key");
+ }
+ my $codes = $ref->[1] // "NULL";
+ push(@letters, "\t{\n\t\t'$key', $children, $codes\n\t}");
+ }
+
+ print "static dm_letter letter_$letter\[\] =\n{\n";
+ for (@letters) {
+ print "$_,\n";
+ }
+ print "\t{\n\t\t'\\0'\n\t}\n";
+ print "};\n";
+}
+
+hash2code($table, '');
+
+
+# Table adapted from https://www.jewishgen.org/InfoFiles/Soundex.html
+#
+# X = NC (not coded)
+#
+# Note that the following letters are coded with substitute letters
+#
+# Ą => a (use '[' for table lookup)
+# Ę => e (use '\\' for table lookup)
+# Ţ => t (use ']' for table lookup)
+#
+# The rule for "UE" below does not correspond to the table referred to above,
+# however it is used by all other known implementations, including the one at
+# https://www.jewishgen.org/jos/jossound.htm (try e.g. "bouey").
+
+__DATA__
+AI,AJ,AY 0,1,X
+AU 0,7,X
+a X,X,6|X,X,X
+A 0,X,X
+B 7,7,7
+CHS 5,54,54
+CH 5,5,5|4,4,4
+CK 5,5,5|45,45,45
+CZ,CS,CSZ,CZS 4,4,4
+C 5,5,5|4,4,4
+DRZ,DRS 4,4,4
+DS,DSH,DSZ 4,4,4
+DZ,DZH,DZS 4,4,4
+D,DT 3,3,3
+EI,EJ,EY 0,1,X
+EU 1,1,X
+e X,X,6|X,X,X
+E 0,X,X
+FB 7,7,7
+F 7,7,7
+G 5,5,5
+H 5,5,X
+IA,IE,IO,IU 1,X,X
+I 0,X,X
+J 1,X,X|4,4,4
+KS 5,54,54
+KH 5,5,5
+K 5,5,5
+L 8,8,8
+MN 66,66,66
+M 6,6,6
+NM 66,66,66
+N 6,6,6
+OI,OJ,OY 0,1,X
+O 0,X,X
+P,PF,PH 7,7,7
+Q 5,5,5
+RZ,RS 94,94,94|4,4,4
+R 9,9,9
+SCHTSCH,SCHTSH,SCHTCH 2,4,4
+SCH 4,4,4
+SHTCH,SHCH,SHTSH 2,4,4
+SHT,SCHT,SCHD 2,43,43
+SH 4,4,4
+STCH,STSCH,SC 2,4,4
+STRZ,STRS,STSH 2,4,4
+ST 2,43,43
+SZCZ,SZCS 2,4,4
+SZT,SHD,SZD,SD 2,43,43
+SZ 4,4,4
+S 4,4,4
+TCH,TTCH,TTSCH 4,4,4
+TH 3,3,3
+TRZ,TRS 4,4,4
+TSCH,TSH 4,4,4
+TS,TTS,TTSZ,TC 4,4,4
+TZ,TTZ,TZS,TSZ 4,4,4
+t 3,3,3|4,4,4
+T 3,3,3
+UI,UJ,UY,UE 0,1,X
+U 0,X,X
+V 7,7,7
+W 7,7,7
+X 5,54,54
+Y 1,X,X
+ZDZ,ZDZH,ZHDZH 2,4,4
+ZD,ZHD 2,43,43
+ZH,ZS,ZSCH,ZSH 4,4,4
+Z 4,4,4
diff --git a/contrib/fuzzystrmatch/expected/fuzzystrmatch.out b/contrib/fuzzystrmatch/expected/fuzzystrmatch.out
index 493c95cdfa..f62ddad4ee 100644
--- a/contrib/fuzzystrmatch/expected/fuzzystrmatch.out
+++ b/contrib/fuzzystrmatch/expected/fuzzystrmatch.out
@@ -65,3 +65,174 @@ SELECT dmetaphone_alt('gumbo');
KMP
(1 row)
+-- Wovels
+SELECT daitch_mokotoff('Augsburg');
+ daitch_mokotoff
+-----------------
+ 054795
+(1 row)
+
+SELECT daitch_mokotoff('Breuer');
+ daitch_mokotoff
+-----------------
+ 791900
+(1 row)
+
+SELECT daitch_mokotoff('Freud');
+ daitch_mokotoff
+-----------------
+ 793000
+(1 row)
+
+-- The letter "H"
+SELECT daitch_mokotoff('Halberstadt');
+ daitch_mokotoff
+-----------------
+ 587943 587433
+(1 row)
+
+SELECT daitch_mokotoff('Mannheim');
+ daitch_mokotoff
+-----------------
+ 665600
+(1 row)
+
+-- Adjacent sounds
+SELECT daitch_mokotoff('Chernowitz');
+ daitch_mokotoff
+-----------------
+ 596740 496740
+(1 row)
+
+-- Adjacent letters with identical adjacent code digits
+SELECT daitch_mokotoff('Cherkassy');
+ daitch_mokotoff
+-----------------
+ 595400 495400
+(1 row)
+
+SELECT daitch_mokotoff('Kleinman');
+ daitch_mokotoff
+-----------------
+ 586660
+(1 row)
+
+-- More than one word
+SELECT daitch_mokotoff('Nowy Targ');
+ daitch_mokotoff
+-----------------
+ 673950
+(1 row)
+
+-- Padded with "0"
+SELECT daitch_mokotoff('Berlin');
+ daitch_mokotoff
+-----------------
+ 798600
+(1 row)
+
+-- Other examples from https://www.avotaynu.com/soundex.htm
+SELECT daitch_mokotoff('Ceniow');
+ daitch_mokotoff
+-----------------
+ 567000 467000
+(1 row)
+
+SELECT daitch_mokotoff('Tsenyuv');
+ daitch_mokotoff
+-----------------
+ 467000
+(1 row)
+
+SELECT daitch_mokotoff('Holubica');
+ daitch_mokotoff
+-----------------
+ 587500 587400
+(1 row)
+
+SELECT daitch_mokotoff('Golubitsa');
+ daitch_mokotoff
+-----------------
+ 587400
+(1 row)
+
+SELECT daitch_mokotoff('Przemysl');
+ daitch_mokotoff
+-----------------
+ 794648 746480
+(1 row)
+
+SELECT daitch_mokotoff('Pshemeshil');
+ daitch_mokotoff
+-----------------
+ 746480
+(1 row)
+
+SELECT daitch_mokotoff('Rosochowaciec');
+ daitch_mokotoff
+---------------------------------------------------------
+ 945755 945754 945745 945744 944755 944754 944745 944744
+(1 row)
+
+SELECT daitch_mokotoff('Rosokhovatsets');
+ daitch_mokotoff
+-----------------
+ 945744
+(1 row)
+
+-- Ignored characters
+SELECT daitch_mokotoff('''OBrien');
+ daitch_mokotoff
+-----------------
+ 079600
+(1 row)
+
+SELECT daitch_mokotoff('O''Brien');
+ daitch_mokotoff
+-----------------
+ 079600
+(1 row)
+
+-- "Difficult" cases, likely to cause trouble for other implementations.
+SELECT daitch_mokotoff('CJC');
+ daitch_mokotoff
+-------------------------------------------
+ 550000 540000 545000 450000 400000 440000
+(1 row)
+
+SELECT daitch_mokotoff('BESST');
+ daitch_mokotoff
+-----------------
+ 743000
+(1 row)
+
+SELECT daitch_mokotoff('BOUEY');
+ daitch_mokotoff
+-----------------
+ 710000
+(1 row)
+
+SELECT daitch_mokotoff('HANNMANN');
+ daitch_mokotoff
+-----------------
+ 566600
+(1 row)
+
+SELECT daitch_mokotoff('MCCOYJR');
+ daitch_mokotoff
+---------------------------------------------------------
+ 651900 654900 654190 654490 645190 645490 641900 644900
+(1 row)
+
+SELECT daitch_mokotoff('ACCURSO');
+ daitch_mokotoff
+---------------------------------------------------------
+ 059400 054000 054940 054400 045940 045400 049400 044000
+(1 row)
+
+SELECT daitch_mokotoff('BIERSCHBACH');
+ daitch_mokotoff
+---------------------------------------------------------
+ 794575 794574 794750 794740 745750 745740 747500 747400
+(1 row)
+
diff --git a/contrib/fuzzystrmatch/expected/fuzzystrmatch_utf8.out b/contrib/fuzzystrmatch/expected/fuzzystrmatch_utf8.out
new file mode 100644
index 0000000000..32d8260383
--- /dev/null
+++ b/contrib/fuzzystrmatch/expected/fuzzystrmatch_utf8.out
@@ -0,0 +1,61 @@
+/*
+ * This test must be run in a database with UTF-8 encoding,
+ * because other encodings don't support all the characters used.
+ */
+SELECT getdatabaseencoding() <> 'UTF8'
+ AS skip_test \gset
+\if :skip_test
+\quit
+\endif
+set client_encoding = utf8;
+-- CREATE EXTENSION IF NOT EXISTS fuzzystrmatch;
+-- Accents
+SELECT daitch_mokotoff('Müller');
+ daitch_mokotoff
+-----------------
+ 689000
+(1 row)
+
+SELECT daitch_mokotoff('Schäfer');
+ daitch_mokotoff
+-----------------
+ 479000
+(1 row)
+
+SELECT daitch_mokotoff('Straßburg');
+ daitch_mokotoff
+-----------------
+ 294795
+(1 row)
+
+SELECT daitch_mokotoff('Éregon');
+ daitch_mokotoff
+-----------------
+ 095600
+(1 row)
+
+-- Special characters added at https://www.jewishgen.org/InfoFiles/Soundex.html
+SELECT daitch_mokotoff('gąszczu');
+ daitch_mokotoff
+-----------------
+ 564000 540000
+(1 row)
+
+SELECT daitch_mokotoff('brzęczy');
+ daitch_mokotoff
+-----------------------------
+ 794640 794400 746400 744000
+(1 row)
+
+SELECT daitch_mokotoff('ţamas');
+ daitch_mokotoff
+-----------------
+ 364000 464000
+(1 row)
+
+SELECT daitch_mokotoff('țamas');
+ daitch_mokotoff
+-----------------
+ 364000 464000
+(1 row)
+
diff --git a/contrib/fuzzystrmatch/expected/fuzzystrmatch_utf8_1.out b/contrib/fuzzystrmatch/expected/fuzzystrmatch_utf8_1.out
new file mode 100644
index 0000000000..37aead89c0
--- /dev/null
+++ b/contrib/fuzzystrmatch/expected/fuzzystrmatch_utf8_1.out
@@ -0,0 +1,8 @@
+/*
+ * This test must be run in a database with UTF-8 encoding,
+ * because other encodings don't support all the characters used.
+ */
+SELECT getdatabaseencoding() <> 'UTF8'
+ AS skip_test \gset
+\if :skip_test
+\quit
diff --git a/contrib/fuzzystrmatch/fuzzystrmatch--1.1--1.2.sql b/contrib/fuzzystrmatch/fuzzystrmatch--1.1--1.2.sql
new file mode 100644
index 0000000000..b9d7b229a3
--- /dev/null
+++ b/contrib/fuzzystrmatch/fuzzystrmatch--1.1--1.2.sql
@@ -0,0 +1,8 @@
+/* contrib/fuzzystrmatch/fuzzystrmatch--1.1--1.2.sql */
+
+-- complain if script is sourced in psql, rather than via ALTER EXTENSION
+\echo Use "ALTER EXTENSION fuzzystrmatch UPDATE TO '1.2'" to load this file. \quit
+
+CREATE FUNCTION daitch_mokotoff(text) RETURNS text
+AS 'MODULE_PATHNAME', 'daitch_mokotoff'
+LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
diff --git a/contrib/fuzzystrmatch/fuzzystrmatch--1.1.sql b/contrib/fuzzystrmatch/fuzzystrmatch--1.2.sql
similarity index 92%
rename from contrib/fuzzystrmatch/fuzzystrmatch--1.1.sql
rename to contrib/fuzzystrmatch/fuzzystrmatch--1.2.sql
index 41de9d949b..2a8a100699 100644
--- a/contrib/fuzzystrmatch/fuzzystrmatch--1.1.sql
+++ b/contrib/fuzzystrmatch/fuzzystrmatch--1.2.sql
@@ -42,3 +42,7 @@ LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
CREATE FUNCTION dmetaphone_alt (text) RETURNS text
AS 'MODULE_PATHNAME', 'dmetaphone_alt'
LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
+
+CREATE FUNCTION daitch_mokotoff(text) RETURNS text
+AS 'MODULE_PATHNAME', 'daitch_mokotoff'
+LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
diff --git a/contrib/fuzzystrmatch/fuzzystrmatch.control b/contrib/fuzzystrmatch/fuzzystrmatch.control
index 3cd6660bf9..8b6e9fd993 100644
--- a/contrib/fuzzystrmatch/fuzzystrmatch.control
+++ b/contrib/fuzzystrmatch/fuzzystrmatch.control
@@ -1,6 +1,6 @@
# fuzzystrmatch extension
comment = 'determine similarities and distance between strings'
-default_version = '1.1'
+default_version = '1.2'
module_pathname = '$libdir/fuzzystrmatch'
relocatable = true
trusted = true
diff --git a/contrib/fuzzystrmatch/sql/fuzzystrmatch.sql b/contrib/fuzzystrmatch/sql/fuzzystrmatch.sql
index f05dc28ffb..db05c7d6b6 100644
--- a/contrib/fuzzystrmatch/sql/fuzzystrmatch.sql
+++ b/contrib/fuzzystrmatch/sql/fuzzystrmatch.sql
@@ -19,3 +19,48 @@ SELECT metaphone('GUMBO', 4);
SELECT dmetaphone('gumbo');
SELECT dmetaphone_alt('gumbo');
+
+-- Wovels
+SELECT daitch_mokotoff('Augsburg');
+SELECT daitch_mokotoff('Breuer');
+SELECT daitch_mokotoff('Freud');
+
+-- The letter "H"
+SELECT daitch_mokotoff('Halberstadt');
+SELECT daitch_mokotoff('Mannheim');
+
+-- Adjacent sounds
+SELECT daitch_mokotoff('Chernowitz');
+
+-- Adjacent letters with identical adjacent code digits
+SELECT daitch_mokotoff('Cherkassy');
+SELECT daitch_mokotoff('Kleinman');
+
+-- More than one word
+SELECT daitch_mokotoff('Nowy Targ');
+
+-- Padded with "0"
+SELECT daitch_mokotoff('Berlin');
+
+-- Other examples from https://www.avotaynu.com/soundex.htm
+SELECT daitch_mokotoff('Ceniow');
+SELECT daitch_mokotoff('Tsenyuv');
+SELECT daitch_mokotoff('Holubica');
+SELECT daitch_mokotoff('Golubitsa');
+SELECT daitch_mokotoff('Przemysl');
+SELECT daitch_mokotoff('Pshemeshil');
+SELECT daitch_mokotoff('Rosochowaciec');
+SELECT daitch_mokotoff('Rosokhovatsets');
+
+-- Ignored characters
+SELECT daitch_mokotoff('''OBrien');
+SELECT daitch_mokotoff('O''Brien');
+
+-- "Difficult" cases, likely to cause trouble for other implementations.
+SELECT daitch_mokotoff('CJC');
+SELECT daitch_mokotoff('BESST');
+SELECT daitch_mokotoff('BOUEY');
+SELECT daitch_mokotoff('HANNMANN');
+SELECT daitch_mokotoff('MCCOYJR');
+SELECT daitch_mokotoff('ACCURSO');
+SELECT daitch_mokotoff('BIERSCHBACH');
diff --git a/contrib/fuzzystrmatch/sql/fuzzystrmatch_utf8.sql b/contrib/fuzzystrmatch/sql/fuzzystrmatch_utf8.sql
new file mode 100644
index 0000000000..f42c01a1bb
--- /dev/null
+++ b/contrib/fuzzystrmatch/sql/fuzzystrmatch_utf8.sql
@@ -0,0 +1,26 @@
+/*
+ * This test must be run in a database with UTF-8 encoding,
+ * because other encodings don't support all the characters used.
+ */
+
+SELECT getdatabaseencoding() <> 'UTF8'
+ AS skip_test \gset
+\if :skip_test
+\quit
+\endif
+
+set client_encoding = utf8;
+
+-- CREATE EXTENSION IF NOT EXISTS fuzzystrmatch;
+
+-- Accents
+SELECT daitch_mokotoff('Müller');
+SELECT daitch_mokotoff('Schäfer');
+SELECT daitch_mokotoff('Straßburg');
+SELECT daitch_mokotoff('Éregon');
+
+-- Special characters added at https://www.jewishgen.org/InfoFiles/Soundex.html
+SELECT daitch_mokotoff('gąszczu');
+SELECT daitch_mokotoff('brzęczy');
+SELECT daitch_mokotoff('ţamas');
+SELECT daitch_mokotoff('țamas');
diff --git a/doc/src/sgml/fuzzystrmatch.sgml b/doc/src/sgml/fuzzystrmatch.sgml
index 382e54be91..08781778f8 100644
--- a/doc/src/sgml/fuzzystrmatch.sgml
+++ b/doc/src/sgml/fuzzystrmatch.sgml
@@ -241,4 +241,101 @@ test=# SELECT dmetaphone('gumbo');
</screen>
</sect2>
+ <sect2>
+ <title>Daitch-Mokotoff Soundex</title>
+
+ <para>
+ Compared to the American Soundex System implemented in the
+ <function>soundex</function> function, the major improvements of the
+ Daitch-Mokotoff Soundex System are:
+
+ <itemizedlist spacing="compact" mark="bullet">
+ <listitem>
+ <para>
+ Information is coded to the first six meaningful letters rather than
+ four.
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ The initial letter is coded rather than kept as is.
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ Where two consecutive letters have a single sound, they are coded as a
+ single number.
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ When a letter or combination of letters may have two different sounds,
+ it is double coded under the two different codes.
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ A letter or combination of letters maps into ten possible codes rather
+ than seven.
+ </para>
+ </listitem>
+ </itemizedlist>
+ </para>
+
+ <indexterm>
+ <primary>daitch_mokotoff</primary>
+ </indexterm>
+
+ <para>
+ The following function generates Daitch-Mokotoff soundex codes for matching
+ of similar-sounding input:
+ </para>
+
+<synopsis>
+daitch_mokotoff(text source) returns text
+</synopsis>
+
+ <para>
+ Since a Daitch-Mokotoff soundex code consists of only 6 digits,
+ <literal>source</literal> should be preferably a single word or name.
+ Any alternative soundex codes are separated by space, which makes the returned
+ text suited for use in Full Text Search, see <xref linkend="textsearch"/> and
+ <xref linkend="functions-textsearch"/>.
+ </para>
+
+ <para>
+ Example:
+ </para>
+
+<programlisting>
+CREATE OR REPLACE FUNCTION soundex_name(v_name text) RETURNS text AS $$
+ SELECT string_agg(daitch_mokotoff(n), ' ')
+ FROM regexp_split_to_table(v_name, '\s+') AS n
+$$ LANGUAGE sql STRICT IMMUTABLE PARALLEL SAFE;
+
+CREATE OR REPLACE FUNCTION soundex_tsvector(v_name text) RETURNS tsvector AS $$
+ SELECT to_tsvector('simple', soundex_name(v_name))
+$$ LANGUAGE sql STRICT IMMUTABLE PARALLEL SAFE;
+
+CREATE OR REPLACE FUNCTION soundex_tsquery(v_name text) RETURNS tsquery AS $$
+ SELECT to_tsquery('simple', quote_literal(soundex_name(v_name)))
+$$ LANGUAGE sql STRICT IMMUTABLE PARALLEL SAFE;
+
+-- Note that searches could be more efficient with the tsvector in a separate column
+-- (no recalculation on table row recheck).
+CREATE TABLE s (nm text);
+CREATE INDEX ix_s_txt ON s USING gin (soundex_tsvector(nm)) WITH (fastupdate = off);
+
+INSERT INTO s VALUES ('John Doe');
+INSERT INTO s VALUES ('Jane Roe');
+INSERT INTO s VALUES ('Public John Q.');
+INSERT INTO s VALUES ('George Best');
+
+SELECT * FROM s WHERE soundex_tsvector(nm) @@ soundex_tsquery('john');
+SELECT * FROM s WHERE soundex_tsvector(nm) @@ soundex_tsquery('jane doe');
+SELECT * FROM s WHERE soundex_tsvector(nm) @@ soundex_tsquery('john public');
+SELECT * FROM s WHERE soundex_tsvector(nm) @@ soundex_tsquery('besst, giorgio');
+</programlisting>
+ </sect2>
+
</sect1>
^ permalink raw reply [nested|flat] 19+ messages in thread
* Re: daitch_mokotoff module
2022-01-03 19:16 Re: daitch_mokotoff module Andres Freund <[email protected]>
2022-01-04 13:49 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
2022-01-04 20:18 ` Re: daitch_mokotoff module Thomas Munro <[email protected]>
2022-01-05 07:05 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
2022-01-05 20:08 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
2022-02-03 14:27 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
@ 2022-11-30 12:56 ` Ian Lawrence Barwick <[email protected]>
2022-12-05 13:24 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
1 sibling, 1 reply; 19+ messages in thread
From: Ian Lawrence Barwick @ 2022-11-30 12:56 UTC (permalink / raw)
To: Dag Lem <[email protected]>; +Cc: PostgreSQL Hackers <[email protected]>; Thomas Munro <[email protected]>; Julien Rouhaud <[email protected]>
Hi Dag
2022年2月3日(木) 23:27 Dag Lem <[email protected]>:
>
> Hi,
>
> Just some minor adjustments to the patch:
>
> * Removed call to locale-dependent toupper()
> * Cleaned up input normalization
This patch was marked as "Waiting on Author" in the CommitFest entry [1]
but I see you provided an updated version which hasn't received any feedback,
so I've move this to the next CommitFest [2] and set it to "Needs Review".
[1] https://commitfest.postgresql.org/40/3451/
[2] https://commitfest.postgresql.org/41/3451/
> I have been asked to sign up to review a commitfest patch or patches -
> unfortunately I've been ill with COVID-19 and it's not until now that
> I feel well enough to have a look.
>
> Julien: I'll have a look at https://commitfest.postgresql.org/36/3468/
> as you suggested (https://commitfest.postgresql.org/36/3379/ seems to
> have been reviewed now).
>
> If there are other suggestions for a patch or patches to review for
> someone new to PostgreSQL internals, I'd be grateful for that.
I see you provided some feedback on https://commitfest.postgresql.org/36/3468/,
though the patch seems to have not been accepted (but not conclusively rejected
either). If you still have the chance to review another patch (or more) it would
be much appreciated, as there's quite a few piling up. Things like documentation
or small improvements to client applications are always a good place to start.
Reviews can be provided at any time, there's no need to wait for the next
CommitFest.
Regards
Ian Barwick
^ permalink raw reply [nested|flat] 19+ messages in thread
* Re: daitch_mokotoff module
2022-01-03 19:16 Re: daitch_mokotoff module Andres Freund <[email protected]>
2022-01-04 13:49 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
2022-01-04 20:18 ` Re: daitch_mokotoff module Thomas Munro <[email protected]>
2022-01-05 07:05 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
2022-01-05 20:08 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
2022-02-03 14:27 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
2022-11-30 12:56 ` Re: daitch_mokotoff module Ian Lawrence Barwick <[email protected]>
@ 2022-12-05 13:24 ` Dag Lem <[email protected]>
2022-12-22 14:02 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
0 siblings, 1 reply; 19+ messages in thread
From: Dag Lem @ 2022-12-05 13:24 UTC (permalink / raw)
To: Ian Lawrence Barwick <[email protected]>; +Cc: PostgreSQL Hackers <[email protected]>; Thomas Munro <[email protected]>; Julien Rouhaud <[email protected]>
Hi Ian,
Ian Lawrence Barwick <[email protected]> writes:
> Hi Dag
>
> 2022年2月3日(木) 23:27 Dag Lem <[email protected]>:
>>
>> Hi,
>>
>> Just some minor adjustments to the patch:
>>
>> * Removed call to locale-dependent toupper()
>> * Cleaned up input normalization
>
> This patch was marked as "Waiting on Author" in the CommitFest entry [1]
> but I see you provided an updated version which hasn't received any feedback,
> so I've move this to the next CommitFest [2] and set it to "Needs Review".
>
> [1] https://commitfest.postgresql.org/40/3451/
> [2] https://commitfest.postgresql.org/41/3451/
>
>> I have been asked to sign up to review a commitfest patch or patches -
>> unfortunately I've been ill with COVID-19 and it's not until now that
>> I feel well enough to have a look.
>>
>> Julien: I'll have a look at https://commitfest.postgresql.org/36/3468/
>> as you suggested (https://commitfest.postgresql.org/36/3379/ seems to
>> have been reviewed now).
>>
>> If there are other suggestions for a patch or patches to review for
>> someone new to PostgreSQL internals, I'd be grateful for that.
>
> I see you provided some feedback on https://commitfest.postgresql.org/36/3468/,
> though the patch seems to have not been accepted (but not conclusively rejected
> either). If you still have the chance to review another patch (or more) it would
> be much appreciated, as there's quite a few piling up. Things like documentation
> or small improvements to client applications are always a good place to start.
> Reviews can be provided at any time, there's no need to wait for the next
> CommitFest.
>
OK, I'll try to find another patch to review.
Regards
Dag Lem
^ permalink raw reply [nested|flat] 19+ messages in thread
* Re: daitch_mokotoff module
2022-01-03 19:16 Re: daitch_mokotoff module Andres Freund <[email protected]>
2022-01-04 13:49 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
2022-01-04 20:18 ` Re: daitch_mokotoff module Thomas Munro <[email protected]>
2022-01-05 07:05 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
2022-01-05 20:08 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
2022-02-03 14:27 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
2022-11-30 12:56 ` Re: daitch_mokotoff module Ian Lawrence Barwick <[email protected]>
2022-12-05 13:24 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
@ 2022-12-22 14:02 ` Dag Lem <[email protected]>
0 siblings, 0 replies; 19+ messages in thread
From: Dag Lem @ 2022-12-22 14:02 UTC (permalink / raw)
To: Ian Lawrence Barwick <[email protected]>; +Cc: PostgreSQL Hackers <[email protected]>; Thomas Munro <[email protected]>; Julien Rouhaud <[email protected]>; Andres Freund <[email protected]>
Dag Lem <[email protected]> writes:
> Hi Ian,
>
> Ian Lawrence Barwick <[email protected]> writes:
>
[...]
>> I see you provided some feedback on
>> https://commitfest.postgresql.org/36/3468/,
>> though the patch seems to have not been accepted (but not
>> conclusively rejected
>> either). If you still have the chance to review another patch (or
>> more) it would
>> be much appreciated, as there's quite a few piling up. Things like
>> documentation
>> or small improvements to client applications are always a good place to start.
>> Reviews can be provided at any time, there's no need to wait for the next
>> CommitFest.
>>
>
> OK, I'll try to find another patch to review.
>
I have scanned through all the patches in Commitfest 2023-01 with status
"Needs review", and it is difficult to find something which I can
meaningfully review.
The only thing I felt qualified to comment (or nit-pick?) on was
https://commitfest.postgresql.org/41/4071/
If something else should turn up which could be reviewed by someone
without intimate knowledge of PostgreSQL internals, then don't hesitate
to ask.
As for the Daitch-Mokotoff patch, the review by Andres Freund was very
helpful in order to improve the extension and to make it more idiomatic
- hopefully it is now a bit closer to being included.
Best regards
Dag Lem
^ permalink raw reply [nested|flat] 19+ messages in thread
* Re: daitch_mokotoff module
2022-01-03 19:16 Re: daitch_mokotoff module Andres Freund <[email protected]>
2022-01-04 13:49 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
2022-01-04 20:18 ` Re: daitch_mokotoff module Thomas Munro <[email protected]>
2022-01-05 07:05 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
2022-01-05 20:08 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
2022-02-03 14:27 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
@ 2022-12-07 18:56 ` Andres Freund <[email protected]>
2022-12-21 09:26 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
1 sibling, 1 reply; 19+ messages in thread
From: Andres Freund @ 2022-12-07 18:56 UTC (permalink / raw)
To: Dag Lem <[email protected]>; +Cc: PostgreSQL Hackers <[email protected]>; Thomas Munro <[email protected]>; Julien Rouhaud <[email protected]>
Hi,
On 2022-02-03 15:27:32 +0100, Dag Lem wrote:
> Just some minor adjustments to the patch:
>
> * Removed call to locale-dependent toupper()
> * Cleaned up input normalization
This patch currently fails in cfbot, likely because meson.build needs to be
adjusted (this didn't exist at the time you submitted this version of the
patch):
[23:43:34.796] contrib/fuzzystrmatch/meson.build:18:0: ERROR: File fuzzystrmatch--1.1.sql does not exist.
> -DATA = fuzzystrmatch--1.1.sql fuzzystrmatch--1.0--1.1.sql
> +DATA = fuzzystrmatch--1.2.sql fuzzystrmatch--1.1--1.2.sql fuzzystrmatch--1.0--1.1.sql
> PGFILEDESC = "fuzzystrmatch - similarities and distance between strings"
The patch seems to remove fuzzystrmatch--1.1.sql - I suggest not doing
that. In recent years our approach has been to just keep the "base version" of
the upgrade script, with extension creation running through the upgrade
scripts.
>
> +
> +#include "daitch_mokotoff.h"
> +
> +#include "postgres.h"
Postgres policy is that the include of "postgres.h" has to be the first
include in every .c file.
> +#include "utils/builtins.h"
> +#include "mb/pg_wchar.h"
> +
> +#include <string.h>
> +
> +/* Internal C implementation */
> +static char *_daitch_mokotoff(char *word, char *soundex, size_t n);
> +
> +
> +PG_FUNCTION_INFO_V1(daitch_mokotoff);
> +Datum
> +daitch_mokotoff(PG_FUNCTION_ARGS)
> +{
> + text *arg = PG_GETARG_TEXT_PP(0);
> + char *string,
> + *tmp_soundex;
> + text *soundex;
> +
> + /*
> + * The maximum theoretical soundex size is several KB, however in practice
> + * anything but contrived synthetic inputs will yield a soundex size of
> + * less than 100 bytes. We thus allocate and free a temporary work buffer,
> + * and return only the actual soundex result.
> + */
> + string = pg_server_to_any(text_to_cstring(arg), VARSIZE_ANY_EXHDR(arg), PG_UTF8);
> + tmp_soundex = palloc(DM_MAX_SOUNDEX_CHARS);
Seems that just using StringInfo to hold the soundex output would work better
than a static allocation?
> + if (!_daitch_mokotoff(string, tmp_soundex, DM_MAX_SOUNDEX_CHARS))
We imo shouldn't introduce new functions starting with _.
> +/* Mark soundex code tree node as leaf. */
> +static void
> +set_leaf(dm_leaves leaves_next, int *num_leaves_next, dm_node * node)
> +{
> + if (!node->is_leaf)
> + {
> + node->is_leaf = 1;
> + leaves_next[(*num_leaves_next)++] = node;
> + }
> +}
> +
> +
> +/* Find next node corresponding to code digit, or create a new node. */
> +static dm_node * find_or_create_node(dm_nodes nodes, int *num_nodes,
> + dm_node * node, char code_digit)
PG code style is to have a line break between a function defintion's return
type and the function name - like you actually do above.
> +/* Mapping from ISO8859-1 to upper-case ASCII */
> +static const char tr_iso8859_1_to_ascii_upper[] =
> +/*
> +"`abcdefghijklmnopqrstuvwxyz{|}~ ¡¢£¤¥¦§¨©ª«¬ ®¯°±²³´µ¶·¸¹º»¼½¾¿ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖ×ØÙÚÛÜÝÞßàáâãäåæçèéêëìíîïðñòóôõö÷øùúûüýþÿ"
> +*/
> +"`ABCDEFGHIJKLMNOPQRSTUVWXYZ{|}~ ! ?AAAAAAECEEEEIIIIDNOOOOO*OUUUUYDSAAAAAAECEEEEIIIIDNOOOOO/OUUUUYDY";
> +
> +static char
> +iso8859_1_to_ascii_upper(unsigned char c)
> +{
> + return c >= 0x60 ? tr_iso8859_1_to_ascii_upper[c - 0x60] : c;
> +}
> +
> +
> +/* Convert an UTF-8 character to ISO-8859-1.
> + * Unconvertable characters are returned as '?'.
> + * NB! Beware of the domain specific conversion of Ą, Ę, and Ţ/Ț.
> + */
> +static char
> +utf8_to_iso8859_1(char *str, int *ix)
It seems decidedly not great to have custom encoding conversion routines in a
contrib module. Is there any way we can avoid this?
> +/* Generate all Daitch-Mokotoff soundex codes for word, separated by space. */
> +static char *
> +_daitch_mokotoff(char *word, char *soundex, size_t n)
> +{
> + int i = 0,
> + j;
> + int letter_no = 0;
> + int ix_leaves = 0;
> + int num_nodes = 0,
> + num_leaves = 0;
> + dm_codes *codes,
> + *next_codes;
> + dm_node *nodes;
> + dm_leaves *leaves;
> +
> + /* First letter. */
> + if (!(codes = read_letter(word, &i)))
> + {
> + /* No encodable character in input. */
> + return NULL;
> + }
> +
> + /* Allocate memory for node tree. */
> + nodes = palloc(sizeof(dm_nodes));
> + leaves = palloc(2 * sizeof(dm_leaves));
So this allocates the worst case memory usage, is that right? That's quite a
bit of memory. Shouldn't nodes be allocated dynamically?
Instead of carefully freeing individual memory allocations, I think it be
better to create a temporary memory context, allocate the necessary nodes etc
on demand, and destroy the temporary memory context at the end.
> +/* Codes for letter sequence at start of name, before a vowel, and any other. */
> +static dm_codes codes_0_1_X[2] =
Any reason these aren't all const?
It's not clear to me where the intended line between the .h and .c file is.
> +print <<EOF;
> +/*
> + * Types and lookup tables for Daitch-Mokotoff Soundex
> + *
If we generate the code, why is the generated header included in the commit?
> +/* Letter in input sequence */
> +struct dm_letter
> +{
> + char letter; /* Present letter in sequence */
> + struct dm_letter *letters; /* List of possible successive letters */
> + dm_codes *codes; /* Code sequence(s) for complete sequence */
> +};
> +
> +/* Node in soundex code tree */
> +struct dm_node
> +{
> + int soundex_length; /* Length of generated soundex code */
> + char soundex[DM_MAX_CODE_DIGITS + 1]; /* Soundex code */
> + int is_leaf; /* Candidate for complete soundex code */
> + int last_update; /* Letter number for last update of node */
> + char code_digit; /* Last code digit, 0 - 9 */
> +
> + /*
> + * One or two alternate code digits leading to this node. If there are two
> + * digits, one of them is always an 'X'. Repeated code digits and 'X' lead
> + * back to the same node.
> + */
> + char prev_code_digits[2];
> + /* One or two alternate code digits moving forward. */
> + char next_code_digits[2];
> + /* ORed together code index(es) used to reach current node. */
> + int prev_code_index;
> + int next_code_index;
> + /* Nodes branching out from this node. */
> + struct dm_node *next_nodes[DM_MAX_ALTERNATE_CODES + 1];
> +};
> +
> +typedef struct dm_letter dm_letter;
> +typedef struct dm_node dm_node;
Why is all this in the generated header? It needs DM_MAX_ALTERNATE_CODES etc,
but it seems that the structs could just be defined in the .c file.
> +# Table adapted from https://www.jewishgen.org/InfoFiles/Soundex.html
What does "adapted" mean here? And what's the path to updating the data?
Greetings,
Andres Freund
^ permalink raw reply [nested|flat] 19+ messages in thread
* Re: daitch_mokotoff module
2022-01-03 19:16 Re: daitch_mokotoff module Andres Freund <[email protected]>
2022-01-04 13:49 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
2022-01-04 20:18 ` Re: daitch_mokotoff module Thomas Munro <[email protected]>
2022-01-05 07:05 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
2022-01-05 20:08 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
2022-02-03 14:27 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
2022-12-07 18:56 ` Re: daitch_mokotoff module Andres Freund <[email protected]>
@ 2022-12-21 09:26 ` Dag Lem <[email protected]>
2022-12-22 12:00 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
0 siblings, 1 reply; 19+ messages in thread
From: Dag Lem @ 2022-12-21 09:26 UTC (permalink / raw)
To: Andres Freund <[email protected]>; +Cc: PostgreSQL Hackers <[email protected]>; Thomas Munro <[email protected]>; Julien Rouhaud <[email protected]>
Hi Andreas,
Thank you for your detailed and constructive review!
I have made a conscientuous effort to address all the issues you point
out, please see comments below.
Andres Freund <[email protected]> writes:
> Hi,
>
> On 2022-02-03 15:27:32 +0100, Dag Lem wrote:
[...]
> [23:43:34.796] contrib/fuzzystrmatch/meson.build:18:0: ERROR: File
> fuzzystrmatch--1.1.sql does not exist.
>
>
>> -DATA = fuzzystrmatch--1.1.sql fuzzystrmatch--1.0--1.1.sql
>> +DATA = fuzzystrmatch--1.2.sql fuzzystrmatch--1.1--1.2.sql
>> fuzzystrmatch--1.0--1.1.sql
>> PGFILEDESC = "fuzzystrmatch - similarities and distance between strings"
>
>
> The patch seems to remove fuzzystrmatch--1.1.sql - I suggest not doing
> that. In recent years our approach has been to just keep the "base version" of
> the upgrade script, with extension creation running through the upgrade
> scripts.
>
OK, I have now kept fuzzystrmatch--1.1.sql, and omitted
fuzzystrmatch--1.2.sql
Both the Makefile and meson.build are updated to handle the new files,
including the generated header.
>>
>> +
>> +#include "daitch_mokotoff.h"
>> +
>> +#include "postgres.h"
>
> Postgres policy is that the include of "postgres.h" has to be the first
> include in every .c file.
>
>
OK, fixed.
>> +#include "utils/builtins.h"
>> +#include "mb/pg_wchar.h"
>> +
>> +#include <string.h>
>> +
>> +/* Internal C implementation */
>> +static char *_daitch_mokotoff(char *word, char *soundex, size_t n);
>> +
>> +
>> +PG_FUNCTION_INFO_V1(daitch_mokotoff);
>> +Datum
>> +daitch_mokotoff(PG_FUNCTION_ARGS)
>> +{
>> + text *arg = PG_GETARG_TEXT_PP(0);
>> + char *string,
>> + *tmp_soundex;
>> + text *soundex;
>> +
>> + /*
>> + * The maximum theoretical soundex size is several KB, however in
>> practice
>> + * anything but contrived synthetic inputs will yield a soundex size of
>> + * less than 100 bytes. We thus allocate and free a temporary work
>> buffer,
>> + * and return only the actual soundex result.
>> + */
>> + string = pg_server_to_any(text_to_cstring(arg),
>> VARSIZE_ANY_EXHDR(arg), PG_UTF8);
>> + tmp_soundex = palloc(DM_MAX_SOUNDEX_CHARS);
>
> Seems that just using StringInfo to hold the soundex output would work better
> than a static allocation?
>
OK, fixed.
>
>> + if (!_daitch_mokotoff(string, tmp_soundex, DM_MAX_SOUNDEX_CHARS))
>
> We imo shouldn't introduce new functions starting with _.
>
OK, fixed. Note that I just followed the existing pattern in
fuzzystrmatch.c there.
[...]
>> +/* Find next node corresponding to code digit, or create a new node. */
>> +static dm_node * find_or_create_node(dm_nodes nodes, int *num_nodes,
>> + dm_node * node, char code_digit)
>
> PG code style is to have a line break between a function defintion's return
> type and the function name - like you actually do above.
>
OK, fixed. Both pgindent and I must have missed that particular
function.
>> +/* Mapping from ISO8859-1 to upper-case ASCII */
>> +static const char tr_iso8859_1_to_ascii_upper[] =
>> +/*
>> +"`abcdefghijklmnopqrstuvwxyz{|}~ ¡¢£¤¥¦§¨©ª«¬
>> ®¯°±²³´µ¶·¸¹º»¼½¾¿ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖ×ØÙÚÛÜÝÞßàáâãäåæçèéêëìíîïðñòóôõö÷øùúûüýþÿ"
>> +*/
>> +"`ABCDEFGHIJKLMNOPQRSTUVWXYZ{|}~ !
>> ?AAAAAAECEEEEIIIIDNOOOOO*OUUUUYDSAAAAAAECEEEEIIIIDNOOOOO/OUUUUYDY";
>> +
>> +static char
>> +iso8859_1_to_ascii_upper(unsigned char c)
>> +{
>> + return c >= 0x60 ? tr_iso8859_1_to_ascii_upper[c - 0x60] : c;
>> +}
>> +
>> +
>> +/* Convert an UTF-8 character to ISO-8859-1.
>> + * Unconvertable characters are returned as '?'.
>> + * NB! Beware of the domain specific conversion of Ą, Ę, and Ţ/Ț.
>> + */
>> +static char
>> +utf8_to_iso8859_1(char *str, int *ix)
>
> It seems decidedly not great to have custom encoding conversion routines in a
> contrib module. Is there any way we can avoid this?
>
I have now replaced the custom UTF-8 decode with calls to
utf8_to_unicode and pg_utf_mblen, and simplified the subsequent
conversion to ASCII. Hopefully this makes the conversion code more
palatable.
I don't see how the conversion to ASCII could be substantially
simplified further. The conversion maps lowercase and 8 bit ISO8859-1
characters to ASCII via uppercasing, removal of accents, and discarding
of special characters. In addition to that, it maps (the non-ISO8859-1)
Ą, Ę, and Ţ/Ț from the coding chart to [, \, and ]. After this, a simple
O(1) table lookup can be used to retrieve the soundex code tree for a
letter sequence.
>
>> +/* Generate all Daitch-Mokotoff soundex codes for word, separated
>> by space. */
>> +static char *
>> +_daitch_mokotoff(char *word, char *soundex, size_t n)
>> +{
>> + int i = 0,
>> + j;
>> + int letter_no = 0;
>> + int ix_leaves = 0;
>> + int num_nodes = 0,
>> + num_leaves = 0;
>> + dm_codes *codes,
>> + *next_codes;
>> + dm_node *nodes;
>> + dm_leaves *leaves;
>> +
>> + /* First letter. */
>> + if (!(codes = read_letter(word, &i)))
>> + {
>> + /* No encodable character in input. */
>> + return NULL;
>> + }
>> +
>> + /* Allocate memory for node tree. */
>> + nodes = palloc(sizeof(dm_nodes));
>> + leaves = palloc(2 * sizeof(dm_leaves));
>
> So this allocates the worst case memory usage, is that right? That's quite a
> bit of memory. Shouldn't nodes be allocated dynamically?
>
> Instead of carefully freeing individual memory allocations, I think it be
> better to create a temporary memory context, allocate the necessary nodes etc
> on demand, and destroy the temporary memory context at the end.
>
Yes, the one-time allocation was intended to cover the worst case memory
usage. This was done to avoid any performance hit incurred by allocating
and deallocating memory for each new node in the soundex code tree.
I have rewritten the bookeeping of nodes in the soundex code tree to use
linked lists, and have followed your advice to use a temporary memory
context for allocation.
I also made an optimization by excluding completed soundex nodes from
the next letter iteration. This seems to offset any allocation overhead
- the performance is more or less the same as before.
>
>> +/* Codes for letter sequence at start of name, before a vowel, and
>> any other. */
>> +static dm_codes codes_0_1_X[2] =
>
> Any reason these aren't all const?
>
No reason why they can't be :-) They are now changed to const.
>
> It's not clear to me where the intended line between the .h and .c file is.
>
>
>> +print <<EOF;
>> +/*
>> + * Types and lookup tables for Daitch-Mokotoff Soundex
>> + *
>
> If we generate the code, why is the generated header included in the commit?
>
This was mainly to have the content available for reference without
having to generate the header. I have removed the file - after the
change you suggest below, the struct declarations are available in the
.c file anyway.
>> +/* Letter in input sequence */
>> +struct dm_letter
>> +{
>> + char letter; /* Present letter in sequence */
>> + struct dm_letter *letters; /* List of possible successive letters
>> */
>> + dm_codes *codes; /* Code sequence(s) for complete sequence */
>> +};
>> +
>> +/* Node in soundex code tree */
>> +struct dm_node
>> +{
>> + int soundex_length; /* Length of generated soundex code */
>> + char soundex[DM_MAX_CODE_DIGITS + 1]; /* Soundex code */
>> + int is_leaf; /* Candidate for complete soundex code */
>> + int last_update; /* Letter number for last update of node */
>> + char code_digit; /* Last code digit, 0 - 9 */
>> +
>> + /*
>> + * One or two alternate code digits leading to this node. If there
>> are two
>> + * digits, one of them is always an 'X'. Repeated code digits and
>> X' lead
>> + * back to the same node.
>> + */
>> + char prev_code_digits[2];
>> + /* One or two alternate code digits moving forward. */
>> + char next_code_digits[2];
>> + /* ORed together code index(es) used to reach current node. */
>> + int prev_code_index;
>> + int next_code_index;
>> + /* Nodes branching out from this node. */
>> + struct dm_node *next_nodes[DM_MAX_ALTERNATE_CODES + 1];
>> +};
>> +
>> +typedef struct dm_letter dm_letter;
>> +typedef struct dm_node dm_node;
>
> Why is all this in the generated header? It needs DM_MAX_ALTERNATE_CODES etc,
> but it seems that the structs could just be defined in the .c file.
>
To accomplish this, I had to rearrange the code a bit. The structs are
now all declared in daitch_mokotoff.c, and the generated header is
included inbetween them.
>
>> +# Table adapted from https://www.jewishgen.org/InfoFiles/Soundex.html
>
> What does "adapted" mean here? And what's the path to updating the data?
>
It means that the original soundex coding chart, which is referred to,
has been converted to a machine readable format, with a few
modifications. These modifications are outlined further down in the
comments. I expanded a bit on the comments, hopefully making things
clearer.
I don't think there is much to be said about updating the data - that's
simply a question of modifying the table and regenerating the header
file. It goes without saying that making changes requires an
understanding of the soundex coding, which is explained in the
reference. However if anything should be unclear, please do point out
what should be explained better.
> Greetings,
>
> Andres Freund
>
Thanks again, and a Merry Christmas to you and all the other PostgreSQL
hackers!
Best regards,
Dag Lem
Attachments:
[text/x-patch] v7-daitch_mokotoff.patch (38.9K, ../../[email protected]/2-v7-daitch_mokotoff.patch)
download | inline diff:
diff --git a/contrib/fuzzystrmatch/Makefile b/contrib/fuzzystrmatch/Makefile
index 0704894f88..12baf2d884 100644
--- a/contrib/fuzzystrmatch/Makefile
+++ b/contrib/fuzzystrmatch/Makefile
@@ -3,14 +3,15 @@
MODULE_big = fuzzystrmatch
OBJS = \
$(WIN32RES) \
+ daitch_mokotoff.o \
dmetaphone.o \
fuzzystrmatch.o
EXTENSION = fuzzystrmatch
-DATA = fuzzystrmatch--1.1.sql fuzzystrmatch--1.0--1.1.sql
+DATA = fuzzystrmatch--1.1.sql fuzzystrmatch--1.1--1.2.sql fuzzystrmatch--1.0--1.1.sql
PGFILEDESC = "fuzzystrmatch - similarities and distance between strings"
-REGRESS = fuzzystrmatch
+REGRESS = fuzzystrmatch fuzzystrmatch_utf8
ifdef USE_PGXS
PG_CONFIG = pg_config
@@ -22,3 +23,14 @@ top_builddir = ../..
include $(top_builddir)/src/Makefile.global
include $(top_srcdir)/contrib/contrib-global.mk
endif
+
+# Force this dependency to be known even without dependency info built:
+daitch_mokotoff.o: daitch_mokotoff.h
+
+daitch_mokotoff.h: daitch_mokotoff_header.pl
+ perl $< $@
+
+distprep: daitch_mokotoff.h
+
+maintainer-clean:
+ rm -f daitch_mokotoff.h
diff --git a/contrib/fuzzystrmatch/daitch_mokotoff.c b/contrib/fuzzystrmatch/daitch_mokotoff.c
new file mode 100644
index 0000000000..d4ad95c283
--- /dev/null
+++ b/contrib/fuzzystrmatch/daitch_mokotoff.c
@@ -0,0 +1,596 @@
+/*
+ * Daitch-Mokotoff Soundex
+ *
+ * Copyright (c) 2021 Finance Norway
+ * Author: Dag Lem <[email protected]>
+ *
+ * This implementation of the Daitch-Mokotoff Soundex System aims at high
+ * performance.
+ *
+ * - The processing of each phoneme is initiated by an O(1) table lookup.
+ * - For phonemes containing more than one character, a coding tree is traversed
+ * to process the complete phoneme.
+ * - The (alternate) soundex codes are produced digit by digit in-place in
+ * another tree structure.
+ *
+ * References:
+ *
+ * https://www.avotaynu.com/soundex.htm
+ * https://www.jewishgen.org/InfoFiles/Soundex.html
+ * https://familypedia.fandom.com/wiki/Daitch-Mokotoff_Soundex
+ * https://stevemorse.org/census/soundex.html (dmlat.php, dmsoundex.php)
+ * https://github.com/apache/commons-codec/ (dmrules.txt, DaitchMokotoffSoundex.java)
+ * https://metacpan.org/pod/Text::Phonetic (DaitchMokotoff.pm)
+ *
+ * A few notes on other implementations:
+ *
+ * - All other known implementations have the same unofficial rules for "UE",
+ * these are also adapted by this implementation (0, 1, NC).
+ * - The only other known implementation which is capable of generating all
+ * correct soundex codes in all cases is the JOS Soundex Calculator at
+ * https://www.jewishgen.org/jos/jossound.htm
+ * - "J" is considered (only) a vowel in dmlat.php
+ * - The official rules for "RS" are commented out in dmlat.php
+ * - Identical code digits for adjacent letters are not collapsed correctly in
+ * dmsoundex.php when double digit codes are involved. E.g. "BESST" yields
+ * 744300 instead of 743000 as for "BEST".
+ * - "J" is considered (only) a consonant in DaitchMokotoffSoundex.java
+ * - "Y" is not considered a vowel in DaitchMokotoffSoundex.java
+ *
+ * Permission to use, copy, modify, and distribute this software and its
+ * documentation for any purpose, without fee, and without a written agreement
+ * is hereby granted, provided that the above copyright notice and this
+ * paragraph and the following two paragraphs appear in all copies.
+ *
+ * IN NO EVENT SHALL THE AUTHOR OR DISTRIBUTORS BE LIABLE TO ANY PARTY FOR
+ * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING
+ * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS
+ * DOCUMENTATION, EVEN IF THE AUTHOR OR DISTRIBUTORS HAVE BEEN ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ * THE AUTHOR AND DISTRIBUTORS SPECIFICALLY DISCLAIM ANY WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
+ * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS
+ * ON AN "AS IS" BASIS, AND THE AUTHOR AND DISTRIBUTORS HAS NO OBLIGATIONS TO
+ * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
+*/
+
+#include "postgres.h"
+
+#include "lib/stringinfo.h"
+#include "mb/pg_wchar.h"
+#include "utils/builtins.h"
+#include "utils/memutils.h"
+
+
+/*
+ * The soundex coding chart table is adapted from from
+ * https://www.jewishgen.org/InfoFiles/Soundex.html
+ * See daitch_mokotoff_header.pl for details.
+*/
+
+#define DM_CODE_DIGITS 6
+
+/* Coding chart table: Soundex codes */
+typedef char dm_code[2 + 1]; /* One or two sequential code digits + NUL */
+typedef dm_code dm_codes[3]; /* Start of name, before a vowel, any other */
+
+/* Coding chart table: Letter in input sequence */
+struct dm_letter
+{
+ char letter; /* Present letter in sequence */
+ const struct dm_letter *letters; /* List of possible successive letters */
+ const dm_codes *codes; /* Code sequence(s) for complete sequence */
+};
+
+typedef struct dm_letter dm_letter;
+
+/* Generated coding chart table */
+#include "daitch_mokotoff.h"
+
+/* Node in soundex code tree */
+struct dm_node
+{
+ int soundex_length; /* Length of generated soundex code */
+ char soundex[DM_CODE_DIGITS + 1]; /* Soundex code */
+ int is_leaf; /* Candidate for complete soundex code */
+ int last_update; /* Letter number for last update of node */
+ char code_digit; /* Last code digit, 0 - 9 */
+
+ /*
+ * One or two alternate code digits leading to this node. If there are two
+ * digits, one of them is always an 'X'. Repeated code digits and 'X' lead
+ * back to the same node.
+ */
+ char prev_code_digits[2];
+ /* One or two alternate code digits moving forward. */
+ char next_code_digits[2];
+ /* ORed together code index(es) used to reach current node. */
+ int prev_code_index;
+ int next_code_index;
+ /* Nodes branching out from this node. */
+ struct dm_node *children[DM_MAX_ALTERNATE_CODES + 1];
+ /* Next node in linked list. Alternating index for each iteration. */
+ struct dm_node *next[2];
+};
+
+typedef struct dm_node dm_node;
+
+
+/* Internal C implementation */
+static int daitch_mokotoff_coding(char *word, StringInfo soundex);
+
+
+PG_FUNCTION_INFO_V1(daitch_mokotoff);
+
+Datum
+daitch_mokotoff(PG_FUNCTION_ARGS)
+{
+ text *arg = PG_GETARG_TEXT_PP(0);
+ char *string;
+ StringInfoData soundex;
+ text *retval;
+ MemoryContext old_ctx,
+ tmp_ctx;
+
+ tmp_ctx = AllocSetContextCreate(CurrentMemoryContext,
+ "daitch_mokotoff temporary context",
+ ALLOCSET_DEFAULT_SIZES);
+ old_ctx = MemoryContextSwitchTo(tmp_ctx);
+
+ string = pg_server_to_any(text_to_cstring(arg), VARSIZE_ANY_EXHDR(arg), PG_UTF8);
+ initStringInfo(&soundex);
+
+ if (!daitch_mokotoff_coding(string, &soundex))
+ {
+ /* No encodable characters in input. */
+ MemoryContextSwitchTo(old_ctx);
+ MemoryContextDelete(tmp_ctx);
+ PG_RETURN_NULL();
+ }
+
+ string = pg_any_to_server(soundex.data, soundex.len, PG_UTF8);
+ MemoryContextSwitchTo(old_ctx);
+ retval = cstring_to_text(string);
+ MemoryContextDelete(tmp_ctx);
+
+ PG_RETURN_TEXT_P(retval);
+}
+
+
+/* Template for new node in soundex code tree. */
+static const dm_node start_node = {
+ .soundex_length = 0,
+ .soundex = "000000 ", /* Six digits + joining space */
+ .is_leaf = 0,
+ .last_update = 0,
+ .code_digit = '\0',
+ .prev_code_digits = {'\0', '\0'},
+ .next_code_digits = {'\0', '\0'},
+ .prev_code_index = 0,
+ .next_code_index = 0,
+ .children = {NULL},
+ .next = {NULL}
+};
+
+/* Dummy soundex codes at end of input. */
+static const dm_codes end_codes[2] =
+{
+ {
+ "X", "X", "X"
+ }
+};
+
+
+/* Initialize soundex code tree node for next code digit. */
+static void
+initialize_node(dm_node * node, int last_update)
+{
+ if (node->last_update < last_update)
+ {
+ node->prev_code_digits[0] = node->next_code_digits[0];
+ node->prev_code_digits[1] = node->next_code_digits[1];
+ node->next_code_digits[0] = '\0';
+ node->next_code_digits[1] = '\0';
+ node->prev_code_index = node->next_code_index;
+ node->next_code_index = 0;
+ node->is_leaf = 0;
+ node->last_update = last_update;
+ }
+}
+
+
+/* Update soundex code tree node with next code digit. */
+static void
+add_next_code_digit(dm_node * node, int code_index, char code_digit)
+{
+ /* OR in index 1 or 2. */
+ node->next_code_index |= code_index;
+
+ if (!node->next_code_digits[0])
+ {
+ node->next_code_digits[0] = code_digit;
+ }
+ else if (node->next_code_digits[0] != code_digit)
+ {
+ node->next_code_digits[1] = code_digit;
+ }
+}
+
+
+/* Mark soundex code tree node as leaf. */
+static void
+set_leaf(dm_node * first_node[2], dm_node * last_node[2], dm_node * node, int ix_node)
+{
+ if (!node->is_leaf)
+ {
+ node->is_leaf = 1;
+
+ if (first_node[ix_node] == NULL)
+ {
+ first_node[ix_node] = node;
+ }
+ else
+ {
+ last_node[ix_node]->next[ix_node] = node;
+ }
+
+ last_node[ix_node] = node;
+ node->next[ix_node] = NULL;
+ }
+}
+
+
+/* Find next node corresponding to code digit, or create a new node. */
+static dm_node *
+find_or_create_child_node(dm_node * parent, char code_digit, StringInfo soundex)
+{
+ dm_node **nodes;
+ dm_node *node;
+ int i;
+
+ for (nodes = parent->children, i = 0; (node = nodes[i]); i++)
+ {
+ if (node->code_digit == code_digit)
+ {
+ /* Found existing child node. Skip completed nodes. */
+ return node->soundex_length < DM_CODE_DIGITS ? node : NULL;
+ }
+ }
+
+ /* Create new child node. */
+ Assert(i < DM_MAX_ALTERNATE_CODES);
+ node = palloc(sizeof(dm_node));
+ nodes[i] = node;
+
+ *node = start_node;
+ memcpy(node->soundex, parent->soundex, sizeof(parent->soundex));
+ node->soundex_length = parent->soundex_length;
+ node->soundex[node->soundex_length++] = code_digit;
+ node->code_digit = code_digit;
+ node->next_code_index = node->prev_code_index;
+
+ if (node->soundex_length < DM_CODE_DIGITS)
+ {
+ return node;
+ }
+ else
+ {
+ /* Append completed soundex code to soundex string. */
+ appendBinaryStringInfoNT(soundex, node->soundex, DM_CODE_DIGITS + 1);
+ return NULL;
+ }
+}
+
+
+/* Update node for next code digit(s). */
+static void
+update_node(dm_node * first_node[2], dm_node * last_node[2], dm_node * node, int ix_node,
+ int letter_no, int prev_code_index, int next_code_index,
+ const char *next_code_digits, int digit_no,
+ StringInfo soundex)
+{
+ int i;
+ char next_code_digit = next_code_digits[digit_no];
+ int num_dirty_nodes = 0;
+ dm_node *dirty_nodes[2];
+
+ initialize_node(node, letter_no);
+
+ if (node->prev_code_index && !(node->prev_code_index & prev_code_index))
+ {
+ /*
+ * If the sound (vowel / consonant) of this letter encoding doesn't
+ * correspond to the coding index of the previous letter, we skip this
+ * letter encoding. Note that currently, only "J" can be either a
+ * vowel or a consonant.
+ */
+ return;
+ }
+
+ if (next_code_digit == 'X' ||
+ (digit_no == 0 &&
+ (node->prev_code_digits[0] == next_code_digit ||
+ node->prev_code_digits[1] == next_code_digit)))
+ {
+ /* The code digit is the same as one of the previous (i.e. not added). */
+ dirty_nodes[num_dirty_nodes++] = node;
+ }
+
+ if (next_code_digit != 'X' &&
+ (digit_no > 0 ||
+ node->prev_code_digits[0] != next_code_digit ||
+ node->prev_code_digits[1]))
+ {
+ /* The code digit is different from one of the previous (i.e. added). */
+ node = find_or_create_child_node(node, next_code_digit, soundex);
+ if (node)
+ {
+ initialize_node(node, letter_no);
+ dirty_nodes[num_dirty_nodes++] = node;
+ }
+ }
+
+ for (i = 0; i < num_dirty_nodes; i++)
+ {
+ /* Add code digit leading to the current node. */
+ add_next_code_digit(dirty_nodes[i], next_code_index, next_code_digit);
+
+ if (next_code_digits[++digit_no])
+ {
+ update_node(first_node, last_node, dirty_nodes[i], ix_node,
+ letter_no, prev_code_index, next_code_index,
+ next_code_digits, digit_no,
+ soundex);
+ }
+ else
+ {
+ /* Add incomplete leaf node to linked list. */
+ set_leaf(first_node, last_node, dirty_nodes[i], ix_node);
+ }
+ }
+}
+
+
+/* Update soundex tree leaf nodes. */
+static void
+update_leaves(dm_node * first_node[2], int *ix_node, int letter_no,
+ const dm_codes * codes, const dm_codes * next_codes,
+ StringInfo soundex)
+{
+ int i,
+ j,
+ code_index;
+ dm_node *node,
+ *last_node[2];
+ const dm_code *code,
+ *next_code;
+ int ix_node_next = (*ix_node + 1) & 1; /* Alternating index: 0, 1 */
+
+ /* Initialize for new linked list of leaves. */
+ first_node[ix_node_next] = NULL;
+ last_node[ix_node_next] = NULL;
+
+ /* Process all nodes. */
+ for (node = first_node[*ix_node]; node; node = node->next[*ix_node])
+ {
+ /* One or two alternate code sequences. */
+ for (i = 0; i < 2 && (code = codes[i]) && code[0][0]; i++)
+ {
+ /* Coding for previous letter - before vowel: 1, all other: 2 */
+ int prev_code_index = (code[0][0] > '1') + 1;
+
+ /* One or two alternate next code sequences. */
+ for (j = 0; j < 2 && (next_code = next_codes[j]) && next_code[0][0]; j++)
+ {
+ /* Determine which code to use. */
+ if (letter_no == 0)
+ {
+ /* This is the first letter. */
+ code_index = 0;
+ }
+ else if (next_code[0][0] <= '1')
+ {
+ /* The next letter is a vowel. */
+ code_index = 1;
+ }
+ else
+ {
+ /* All other cases. */
+ code_index = 2;
+ }
+
+ /* One or two sequential code digits. */
+ update_node(first_node, last_node, node, ix_node_next,
+ letter_no, prev_code_index, code_index,
+ code[code_index], 0,
+ soundex);
+ }
+ }
+ }
+
+ *ix_node = ix_node_next;
+}
+
+
+/* Mapping from ISO8859-1 to upper-case ASCII */
+static const char iso8859_1_to_ascii_upper[] =
+/*
+"`abcdefghijklmnopqrstuvwxyz{|}~ ¡¢£¤¥¦§¨©ª«¬ ®¯°±²³´µ¶·¸¹º»¼½¾¿ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖ×ØÙÚÛÜÝÞßàáâãäåæçèéêëìíîïðñòóôõö÷øùúûüýþÿ"
+*/
+"`ABCDEFGHIJKLMNOPQRSTUVWXYZ{|}~ ! ?AAAAAAECEEEEIIIIDNOOOOO*OUUUUYDSAAAAAAECEEEEIIIIDNOOOOO/OUUUUYDY";
+
+
+/* Return next character, converted from UTF-8 to uppercase ASCII. */
+static char
+read_char(unsigned char *str, int *ix)
+{
+ /* Substitute character for skipped code points. */
+ const char na = '\x1a';
+ pg_wchar c;
+
+ /* Decode UTF-8 character to ISO 10646 code point. */
+ str += *ix;
+ c = utf8_to_unicode(str);
+ *ix += pg_utf_mblen(str);
+
+ if (c >= (unsigned char) '[' && c <= (unsigned char) ']')
+ {
+ /* ASCII characters [, \, and ] are reserved for Ą, Ę, and Ţ/Ț. */
+ return na;
+ }
+ else if (c < 0x60)
+ {
+ /* Non-lowercase ASCII character. */
+ return c;
+ }
+ else if (c < 0x100)
+ {
+ /* ISO-8859-1 code point, converted to upper-case ASCII. */
+ return iso8859_1_to_ascii_upper[c - 0x60];
+ }
+ else
+ {
+ /* Conversion of non-ASCII characters in the coding chart. */
+ switch (c)
+ {
+ case 0x0104:
+ case 0x0105:
+ /* Ą/ą */
+ return '[';
+ case 0x0118:
+ case 0x0119:
+ /* Ę/ę */
+ return '\\';
+ case 0x0162:
+ case 0x0163:
+ case 0x021A:
+ case 0x021B:
+ /* Ţ/ţ or Ț/ț */
+ return ']';
+ default:
+ return na;
+ }
+ }
+}
+
+
+/* Read next ASCII character, skipping any characters not in [A-\]]. */
+static char
+read_valid_char(char *str, int *ix)
+{
+ char c;
+
+ while ((c = read_char((unsigned char *) str, ix)))
+ {
+ if (c >= 'A' && c <= ']')
+ {
+ break;
+ }
+ }
+
+ return c;
+}
+
+
+/* Return sound coding for "letter" (letter sequence) */
+static const dm_codes *
+read_letter(char *str, int *ix)
+{
+ char c,
+ cmp;
+ int i,
+ j;
+ const dm_letter *letters;
+ const dm_codes *codes;
+
+ /* First letter in sequence. */
+ if (!(c = read_valid_char(str, ix)))
+ {
+ return NULL;
+ }
+ letters = &letter_[c - 'A'];
+ codes = letters->codes;
+ i = *ix;
+
+ /* Any subsequent letters in sequence. */
+ while ((letters = letters->letters) && (c = read_valid_char(str, &i)))
+ {
+ for (j = 0; (cmp = letters[j].letter); j++)
+ {
+ if (cmp == c)
+ {
+ /* Letter found. */
+ letters = &letters[j];
+ if (letters->codes)
+ {
+ /* Coding for letter sequence found. */
+ codes = letters->codes;
+ *ix = i;
+ }
+ break;
+ }
+ }
+ if (!cmp)
+ {
+ /* The sequence of letters has no coding. */
+ break;
+ }
+ }
+
+ return codes;
+}
+
+
+/* Generate all Daitch-Mokotoff soundex codes for word, separated by space. */
+static int
+daitch_mokotoff_coding(char *word, StringInfo soundex)
+{
+ int i = 0;
+ int letter_no = 0;
+ int ix_node = 0;
+ const dm_codes *codes,
+ *next_codes;
+ dm_node *first_node[2],
+ *node;
+
+ /* First letter. */
+ if (!(codes = read_letter(word, &i)))
+ {
+ /* No encodable character in input. */
+ return 0;
+ }
+
+ /* Starting point. */
+ first_node[ix_node] = palloc(sizeof(dm_node));
+ *first_node[ix_node] = start_node;
+
+ /*
+ * Loop until either the word input is exhausted, or all generated soundex
+ * codes are completed to six digits.
+ */
+ while (codes && first_node[ix_node])
+ {
+ next_codes = read_letter(word, &i);
+
+ /* Update leaf nodes. */
+ update_leaves(first_node, &ix_node, letter_no,
+ codes, next_codes ? next_codes : end_codes,
+ soundex);
+
+ codes = next_codes;
+ letter_no++;
+ }
+
+ /* Append all remaining (incomplete) soundex codes. */
+ for (node = first_node[ix_node]; node; node = node->next[ix_node])
+ {
+ appendBinaryStringInfoNT(soundex, node->soundex, DM_CODE_DIGITS + 1);
+ }
+
+ /* Terminate string at the final space. */
+ soundex->len--;
+ soundex->data[soundex->len] = '\0';
+
+ return 1;
+}
diff --git a/contrib/fuzzystrmatch/daitch_mokotoff_header.pl b/contrib/fuzzystrmatch/daitch_mokotoff_header.pl
new file mode 100755
index 0000000000..807b5fb8c5
--- /dev/null
+++ b/contrib/fuzzystrmatch/daitch_mokotoff_header.pl
@@ -0,0 +1,260 @@
+#!/bin/perl
+#
+# Generation of types and lookup tables for Daitch-Mokotoff soundex.
+#
+# Copyright (c) 2021 Finance Norway
+# Author: Dag Lem <[email protected]>
+#
+# Permission to use, copy, modify, and distribute this software and its
+# documentation for any purpose, without fee, and without a written agreement
+# is hereby granted, provided that the above copyright notice and this
+# paragraph and the following two paragraphs appear in all copies.
+#
+# IN NO EVENT SHALL THE AUTHOR OR DISTRIBUTORS BE LIABLE TO ANY PARTY FOR
+# DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING
+# LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS
+# DOCUMENTATION, EVEN IF THE AUTHOR OR DISTRIBUTORS HAVE BEEN ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+#
+# THE AUTHOR AND DISTRIBUTORS SPECIFICALLY DISCLAIM ANY WARRANTIES,
+# INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
+# AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS
+# ON AN "AS IS" BASIS, AND THE AUTHOR AND DISTRIBUTORS HAS NO OBLIGATIONS TO
+# PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
+#
+
+use strict;
+use warnings;
+use utf8;
+use open IO => ':utf8', ':std';
+use Data::Dumper;
+
+die "Usage: $0 OUTPUT_FILE\n" if @ARGV != 1;
+my $output_file = $ARGV[0];
+
+# Open the output file
+open my $OUTPUT, '>', $output_file
+ or die "Could not open output file $output_file: $!\n";
+
+# Parse code table and generate tree for letter transitions.
+my %codes;
+my %alternates;
+my $table = [{}, [["","",""]]];
+while (<DATA>) {
+ chomp;
+ my ($letters, $codes) = split(/\s+/);
+ my @codes = map { [ split(/,/) ] } split(/\|/, $codes);
+
+ # Find alternate code transitions for calculation of storage.
+ # The first character ("start of a name") can never yield more than two alternate codes,
+ # and is not considered here.
+ if (@codes > 1) {
+ for my $j (1..2) { # Codes for "before a vowel" and "any other"
+ for my $i (0..1) { # Alternate codes
+ # Identical code digits for adjacent letters are collapsed.
+ # For each possible non-transition due to code digit
+ # collapsing, find all alternate transitions.
+ my ($present, $next) = ($codes[$i][$j], $codes[($i + 1)%2][$j]);
+ next if length($present) != 1;
+ $next = $present ne substr($next, 0, 1) ? substr($next, 0, 1) : substr($next, -1, 1);
+ $alternates{$present}{$next} = 1;
+ }
+ }
+ }
+ my $key = "codes_" . join("_or_", map { join("_", @$_) } @codes);
+ my $val = join(",\n", map { "\t{\n\t\t" . join(", ", map { "\"$_\"" } @$_) . "\n\t}" } @codes);
+ $codes{$key} = $val;
+
+ for my $letter (split(/,/, $letters)) {
+ my $ref = $table->[0];
+ # Link each character to the next in the letter combination.
+ my @c = split(//, $letter);
+ my $last_c = pop(@c);
+ for my $c (@c) {
+ $ref->{$c} //= [ {}, undef ];
+ $ref->{$c}[0] //= {};
+ $ref = $ref->{$c}[0];
+ }
+ # The sound code for the letter combination is stored at the last character.
+ $ref->{$last_c}[1] = $key;
+ }
+}
+close(DATA);
+
+# Add alternates by following transitions to 'X' (not coded).
+my $alt_x = $alternates{"X"};
+delete $alt_x->{"X"};
+while (my ($k, $v) = each %alternates) {
+ if (delete $v->{"X"}) {
+ for my $x (keys %$alt_x) {
+ $v->{$x} = 1;
+ }
+ }
+}
+
+# Find the maximum number of alternate codes in one position.
+# Add two for any additional final code digit transitions.
+my $max_alt = (sort { $b <=> $a } (map { scalar keys %$_ } values %alternates))[0] + 2;
+
+print $OUTPUT <<EOF;
+/*
+ * Constants and lookup tables for Daitch-Mokotoff Soundex
+ *
+ * Copyright (c) 2021 Finance Norway
+ * Author: Dag Lem <dag\@nimrod.no>
+ *
+ * This file is generated by daitch_mokotoff_header.pl
+ *
+ * Permission to use, copy, modify, and distribute this software and its
+ * documentation for any purpose, without fee, and without a written agreement
+ * is hereby granted, provided that the above copyright notice and this
+ * paragraph and the following two paragraphs appear in all copies.
+ *
+ * IN NO EVENT SHALL THE AUTHOR OR DISTRIBUTORS BE LIABLE TO ANY PARTY FOR
+ * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING
+ * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS
+ * DOCUMENTATION, EVEN IF THE AUTHOR OR DISTRIBUTORS HAVE BEEN ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ * THE AUTHOR AND DISTRIBUTORS SPECIFICALLY DISCLAIM ANY WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
+ * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS
+ * ON AN "AS IS" BASIS, AND THE AUTHOR AND DISTRIBUTORS HAS NO OBLIGATIONS TO
+ * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
+ */
+
+#define DM_MAX_ALTERNATE_CODES $max_alt
+
+/* Codes for letter sequence at start of name, before a vowel, and any other. */
+EOF
+
+for my $key (sort keys %codes) {
+ print $OUTPUT "static const dm_codes $key\[2\] =\n{\n" . $codes{$key} . "\n};\n";
+}
+
+print $OUTPUT <<EOF;
+
+/* Coding for alternative following letters in sequence. */
+EOF
+
+sub hash2code {
+ my ($ref, $letter) = @_;
+
+ my @letters = ();
+
+ my $h = $ref->[0];
+ for my $key (sort keys %$h) {
+ $ref = $h->{$key};
+ my $children = "NULL";
+ if (defined $ref->[0]) {
+ $children = "letter_$letter$key";
+ hash2code($ref, "$letter$key");
+ }
+ my $codes = $ref->[1] // "NULL";
+ push(@letters, "\t{\n\t\t'$key', $children, $codes\n\t}");
+ }
+
+ print $OUTPUT "static const dm_letter letter_$letter\[\] =\n{\n";
+ for (@letters) {
+ print $OUTPUT "$_,\n";
+ }
+ print $OUTPUT "\t{\n\t\t'\\0'\n\t}\n";
+ print $OUTPUT "};\n";
+}
+
+hash2code($table, '');
+
+close $OUTPUT;
+
+# Table adapted from https://www.jewishgen.org/InfoFiles/Soundex.html
+#
+# The conversion from the coding chart to the table should be self
+# explanatory, but note the differences stated below.
+#
+# X = NC (not coded)
+#
+# The non-ASCII letters in the coding chart are coded with substitute
+# lowercase ASCII letters, which sort after the uppercase ASCII letters:
+#
+# Ą => a (use '[' for table lookup)
+# Ę => e (use '\\' for table lookup)
+# Ţ => t (use ']' for table lookup)
+#
+# The rule for "UE" does not correspond to the coding chart, however
+# it is used by all other known implementations, including the one at
+# https://www.jewishgen.org/jos/jossound.htm (try e.g. "bouey").
+#
+# Note that the implementation assumes that vowels are assigned code
+# 0 or 1. "J" can be either a vowel or a consonant.
+#
+
+__DATA__
+AI,AJ,AY 0,1,X
+AU 0,7,X
+a X,X,6|X,X,X
+A 0,X,X
+B 7,7,7
+CHS 5,54,54
+CH 5,5,5|4,4,4
+CK 5,5,5|45,45,45
+CZ,CS,CSZ,CZS 4,4,4
+C 5,5,5|4,4,4
+DRZ,DRS 4,4,4
+DS,DSH,DSZ 4,4,4
+DZ,DZH,DZS 4,4,4
+D,DT 3,3,3
+EI,EJ,EY 0,1,X
+EU 1,1,X
+e X,X,6|X,X,X
+E 0,X,X
+FB 7,7,7
+F 7,7,7
+G 5,5,5
+H 5,5,X
+IA,IE,IO,IU 1,X,X
+I 0,X,X
+J 1,X,X|4,4,4
+KS 5,54,54
+KH 5,5,5
+K 5,5,5
+L 8,8,8
+MN 66,66,66
+M 6,6,6
+NM 66,66,66
+N 6,6,6
+OI,OJ,OY 0,1,X
+O 0,X,X
+P,PF,PH 7,7,7
+Q 5,5,5
+RZ,RS 94,94,94|4,4,4
+R 9,9,9
+SCHTSCH,SCHTSH,SCHTCH 2,4,4
+SCH 4,4,4
+SHTCH,SHCH,SHTSH 2,4,4
+SHT,SCHT,SCHD 2,43,43
+SH 4,4,4
+STCH,STSCH,SC 2,4,4
+STRZ,STRS,STSH 2,4,4
+ST 2,43,43
+SZCZ,SZCS 2,4,4
+SZT,SHD,SZD,SD 2,43,43
+SZ 4,4,4
+S 4,4,4
+TCH,TTCH,TTSCH 4,4,4
+TH 3,3,3
+TRZ,TRS 4,4,4
+TSCH,TSH 4,4,4
+TS,TTS,TTSZ,TC 4,4,4
+TZ,TTZ,TZS,TSZ 4,4,4
+t 3,3,3|4,4,4
+T 3,3,3
+UI,UJ,UY,UE 0,1,X
+U 0,X,X
+V 7,7,7
+W 7,7,7
+X 5,54,54
+Y 1,X,X
+ZDZ,ZDZH,ZHDZH 2,4,4
+ZD,ZHD 2,43,43
+ZH,ZS,ZSCH,ZSH 4,4,4
+Z 4,4,4
diff --git a/contrib/fuzzystrmatch/expected/fuzzystrmatch.out b/contrib/fuzzystrmatch/expected/fuzzystrmatch.out
index 493c95cdfa..f62ddad4ee 100644
--- a/contrib/fuzzystrmatch/expected/fuzzystrmatch.out
+++ b/contrib/fuzzystrmatch/expected/fuzzystrmatch.out
@@ -65,3 +65,174 @@ SELECT dmetaphone_alt('gumbo');
KMP
(1 row)
+-- Wovels
+SELECT daitch_mokotoff('Augsburg');
+ daitch_mokotoff
+-----------------
+ 054795
+(1 row)
+
+SELECT daitch_mokotoff('Breuer');
+ daitch_mokotoff
+-----------------
+ 791900
+(1 row)
+
+SELECT daitch_mokotoff('Freud');
+ daitch_mokotoff
+-----------------
+ 793000
+(1 row)
+
+-- The letter "H"
+SELECT daitch_mokotoff('Halberstadt');
+ daitch_mokotoff
+-----------------
+ 587943 587433
+(1 row)
+
+SELECT daitch_mokotoff('Mannheim');
+ daitch_mokotoff
+-----------------
+ 665600
+(1 row)
+
+-- Adjacent sounds
+SELECT daitch_mokotoff('Chernowitz');
+ daitch_mokotoff
+-----------------
+ 596740 496740
+(1 row)
+
+-- Adjacent letters with identical adjacent code digits
+SELECT daitch_mokotoff('Cherkassy');
+ daitch_mokotoff
+-----------------
+ 595400 495400
+(1 row)
+
+SELECT daitch_mokotoff('Kleinman');
+ daitch_mokotoff
+-----------------
+ 586660
+(1 row)
+
+-- More than one word
+SELECT daitch_mokotoff('Nowy Targ');
+ daitch_mokotoff
+-----------------
+ 673950
+(1 row)
+
+-- Padded with "0"
+SELECT daitch_mokotoff('Berlin');
+ daitch_mokotoff
+-----------------
+ 798600
+(1 row)
+
+-- Other examples from https://www.avotaynu.com/soundex.htm
+SELECT daitch_mokotoff('Ceniow');
+ daitch_mokotoff
+-----------------
+ 567000 467000
+(1 row)
+
+SELECT daitch_mokotoff('Tsenyuv');
+ daitch_mokotoff
+-----------------
+ 467000
+(1 row)
+
+SELECT daitch_mokotoff('Holubica');
+ daitch_mokotoff
+-----------------
+ 587500 587400
+(1 row)
+
+SELECT daitch_mokotoff('Golubitsa');
+ daitch_mokotoff
+-----------------
+ 587400
+(1 row)
+
+SELECT daitch_mokotoff('Przemysl');
+ daitch_mokotoff
+-----------------
+ 794648 746480
+(1 row)
+
+SELECT daitch_mokotoff('Pshemeshil');
+ daitch_mokotoff
+-----------------
+ 746480
+(1 row)
+
+SELECT daitch_mokotoff('Rosochowaciec');
+ daitch_mokotoff
+---------------------------------------------------------
+ 945755 945754 945745 945744 944755 944754 944745 944744
+(1 row)
+
+SELECT daitch_mokotoff('Rosokhovatsets');
+ daitch_mokotoff
+-----------------
+ 945744
+(1 row)
+
+-- Ignored characters
+SELECT daitch_mokotoff('''OBrien');
+ daitch_mokotoff
+-----------------
+ 079600
+(1 row)
+
+SELECT daitch_mokotoff('O''Brien');
+ daitch_mokotoff
+-----------------
+ 079600
+(1 row)
+
+-- "Difficult" cases, likely to cause trouble for other implementations.
+SELECT daitch_mokotoff('CJC');
+ daitch_mokotoff
+-------------------------------------------
+ 550000 540000 545000 450000 400000 440000
+(1 row)
+
+SELECT daitch_mokotoff('BESST');
+ daitch_mokotoff
+-----------------
+ 743000
+(1 row)
+
+SELECT daitch_mokotoff('BOUEY');
+ daitch_mokotoff
+-----------------
+ 710000
+(1 row)
+
+SELECT daitch_mokotoff('HANNMANN');
+ daitch_mokotoff
+-----------------
+ 566600
+(1 row)
+
+SELECT daitch_mokotoff('MCCOYJR');
+ daitch_mokotoff
+---------------------------------------------------------
+ 651900 654900 654190 654490 645190 645490 641900 644900
+(1 row)
+
+SELECT daitch_mokotoff('ACCURSO');
+ daitch_mokotoff
+---------------------------------------------------------
+ 059400 054000 054940 054400 045940 045400 049400 044000
+(1 row)
+
+SELECT daitch_mokotoff('BIERSCHBACH');
+ daitch_mokotoff
+---------------------------------------------------------
+ 794575 794574 794750 794740 745750 745740 747500 747400
+(1 row)
+
diff --git a/contrib/fuzzystrmatch/expected/fuzzystrmatch_utf8.out b/contrib/fuzzystrmatch/expected/fuzzystrmatch_utf8.out
new file mode 100644
index 0000000000..32d8260383
--- /dev/null
+++ b/contrib/fuzzystrmatch/expected/fuzzystrmatch_utf8.out
@@ -0,0 +1,61 @@
+/*
+ * This test must be run in a database with UTF-8 encoding,
+ * because other encodings don't support all the characters used.
+ */
+SELECT getdatabaseencoding() <> 'UTF8'
+ AS skip_test \gset
+\if :skip_test
+\quit
+\endif
+set client_encoding = utf8;
+-- CREATE EXTENSION IF NOT EXISTS fuzzystrmatch;
+-- Accents
+SELECT daitch_mokotoff('Müller');
+ daitch_mokotoff
+-----------------
+ 689000
+(1 row)
+
+SELECT daitch_mokotoff('Schäfer');
+ daitch_mokotoff
+-----------------
+ 479000
+(1 row)
+
+SELECT daitch_mokotoff('Straßburg');
+ daitch_mokotoff
+-----------------
+ 294795
+(1 row)
+
+SELECT daitch_mokotoff('Éregon');
+ daitch_mokotoff
+-----------------
+ 095600
+(1 row)
+
+-- Special characters added at https://www.jewishgen.org/InfoFiles/Soundex.html
+SELECT daitch_mokotoff('gąszczu');
+ daitch_mokotoff
+-----------------
+ 564000 540000
+(1 row)
+
+SELECT daitch_mokotoff('brzęczy');
+ daitch_mokotoff
+-----------------------------
+ 794640 794400 746400 744000
+(1 row)
+
+SELECT daitch_mokotoff('ţamas');
+ daitch_mokotoff
+-----------------
+ 364000 464000
+(1 row)
+
+SELECT daitch_mokotoff('țamas');
+ daitch_mokotoff
+-----------------
+ 364000 464000
+(1 row)
+
diff --git a/contrib/fuzzystrmatch/expected/fuzzystrmatch_utf8_1.out b/contrib/fuzzystrmatch/expected/fuzzystrmatch_utf8_1.out
new file mode 100644
index 0000000000..37aead89c0
--- /dev/null
+++ b/contrib/fuzzystrmatch/expected/fuzzystrmatch_utf8_1.out
@@ -0,0 +1,8 @@
+/*
+ * This test must be run in a database with UTF-8 encoding,
+ * because other encodings don't support all the characters used.
+ */
+SELECT getdatabaseencoding() <> 'UTF8'
+ AS skip_test \gset
+\if :skip_test
+\quit
diff --git a/contrib/fuzzystrmatch/fuzzystrmatch--1.1--1.2.sql b/contrib/fuzzystrmatch/fuzzystrmatch--1.1--1.2.sql
new file mode 100644
index 0000000000..b9d7b229a3
--- /dev/null
+++ b/contrib/fuzzystrmatch/fuzzystrmatch--1.1--1.2.sql
@@ -0,0 +1,8 @@
+/* contrib/fuzzystrmatch/fuzzystrmatch--1.1--1.2.sql */
+
+-- complain if script is sourced in psql, rather than via ALTER EXTENSION
+\echo Use "ALTER EXTENSION fuzzystrmatch UPDATE TO '1.2'" to load this file. \quit
+
+CREATE FUNCTION daitch_mokotoff(text) RETURNS text
+AS 'MODULE_PATHNAME', 'daitch_mokotoff'
+LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
diff --git a/contrib/fuzzystrmatch/fuzzystrmatch.control b/contrib/fuzzystrmatch/fuzzystrmatch.control
index 3cd6660bf9..8b6e9fd993 100644
--- a/contrib/fuzzystrmatch/fuzzystrmatch.control
+++ b/contrib/fuzzystrmatch/fuzzystrmatch.control
@@ -1,6 +1,6 @@
# fuzzystrmatch extension
comment = 'determine similarities and distance between strings'
-default_version = '1.1'
+default_version = '1.2'
module_pathname = '$libdir/fuzzystrmatch'
relocatable = true
trusted = true
diff --git a/contrib/fuzzystrmatch/meson.build b/contrib/fuzzystrmatch/meson.build
index e6d06149ce..6b4a13694f 100644
--- a/contrib/fuzzystrmatch/meson.build
+++ b/contrib/fuzzystrmatch/meson.build
@@ -1,7 +1,16 @@
fuzzystrmatch_sources = files(
- 'fuzzystrmatch.c',
+ 'daitch_mokotoff.c',
'dmetaphone.c',
+ 'fuzzystrmatch.c',
+)
+
+daitch_mokotoff_h = custom_target('daitch_mokotoff',
+ input: 'daitch_mokotoff_header.pl',
+ output: 'daitch_mokotoff.h',
+ command: [perl, '@INPUT@', '@OUTPUT@'],
)
+generated_sources += daitch_mokotoff_h
+fuzzystrmatch_sources += daitch_mokotoff_h
if host_system == 'windows'
fuzzystrmatch_sources += rc_lib_gen.process(win32ver_rc, extra_args: [
@@ -19,6 +28,7 @@ install_data(
'fuzzystrmatch.control',
'fuzzystrmatch--1.0--1.1.sql',
'fuzzystrmatch--1.1.sql',
+ 'fuzzystrmatch--1.1--1.2.sql',
kwargs: contrib_data_args,
)
@@ -29,6 +39,7 @@ tests += {
'regress': {
'sql': [
'fuzzystrmatch',
+ 'fuzzystrmatch_utf8',
],
},
}
diff --git a/contrib/fuzzystrmatch/sql/fuzzystrmatch.sql b/contrib/fuzzystrmatch/sql/fuzzystrmatch.sql
index f05dc28ffb..db05c7d6b6 100644
--- a/contrib/fuzzystrmatch/sql/fuzzystrmatch.sql
+++ b/contrib/fuzzystrmatch/sql/fuzzystrmatch.sql
@@ -19,3 +19,48 @@ SELECT metaphone('GUMBO', 4);
SELECT dmetaphone('gumbo');
SELECT dmetaphone_alt('gumbo');
+
+-- Wovels
+SELECT daitch_mokotoff('Augsburg');
+SELECT daitch_mokotoff('Breuer');
+SELECT daitch_mokotoff('Freud');
+
+-- The letter "H"
+SELECT daitch_mokotoff('Halberstadt');
+SELECT daitch_mokotoff('Mannheim');
+
+-- Adjacent sounds
+SELECT daitch_mokotoff('Chernowitz');
+
+-- Adjacent letters with identical adjacent code digits
+SELECT daitch_mokotoff('Cherkassy');
+SELECT daitch_mokotoff('Kleinman');
+
+-- More than one word
+SELECT daitch_mokotoff('Nowy Targ');
+
+-- Padded with "0"
+SELECT daitch_mokotoff('Berlin');
+
+-- Other examples from https://www.avotaynu.com/soundex.htm
+SELECT daitch_mokotoff('Ceniow');
+SELECT daitch_mokotoff('Tsenyuv');
+SELECT daitch_mokotoff('Holubica');
+SELECT daitch_mokotoff('Golubitsa');
+SELECT daitch_mokotoff('Przemysl');
+SELECT daitch_mokotoff('Pshemeshil');
+SELECT daitch_mokotoff('Rosochowaciec');
+SELECT daitch_mokotoff('Rosokhovatsets');
+
+-- Ignored characters
+SELECT daitch_mokotoff('''OBrien');
+SELECT daitch_mokotoff('O''Brien');
+
+-- "Difficult" cases, likely to cause trouble for other implementations.
+SELECT daitch_mokotoff('CJC');
+SELECT daitch_mokotoff('BESST');
+SELECT daitch_mokotoff('BOUEY');
+SELECT daitch_mokotoff('HANNMANN');
+SELECT daitch_mokotoff('MCCOYJR');
+SELECT daitch_mokotoff('ACCURSO');
+SELECT daitch_mokotoff('BIERSCHBACH');
diff --git a/contrib/fuzzystrmatch/sql/fuzzystrmatch_utf8.sql b/contrib/fuzzystrmatch/sql/fuzzystrmatch_utf8.sql
new file mode 100644
index 0000000000..f42c01a1bb
--- /dev/null
+++ b/contrib/fuzzystrmatch/sql/fuzzystrmatch_utf8.sql
@@ -0,0 +1,26 @@
+/*
+ * This test must be run in a database with UTF-8 encoding,
+ * because other encodings don't support all the characters used.
+ */
+
+SELECT getdatabaseencoding() <> 'UTF8'
+ AS skip_test \gset
+\if :skip_test
+\quit
+\endif
+
+set client_encoding = utf8;
+
+-- CREATE EXTENSION IF NOT EXISTS fuzzystrmatch;
+
+-- Accents
+SELECT daitch_mokotoff('Müller');
+SELECT daitch_mokotoff('Schäfer');
+SELECT daitch_mokotoff('Straßburg');
+SELECT daitch_mokotoff('Éregon');
+
+-- Special characters added at https://www.jewishgen.org/InfoFiles/Soundex.html
+SELECT daitch_mokotoff('gąszczu');
+SELECT daitch_mokotoff('brzęczy');
+SELECT daitch_mokotoff('ţamas');
+SELECT daitch_mokotoff('țamas');
diff --git a/doc/src/sgml/fuzzystrmatch.sgml b/doc/src/sgml/fuzzystrmatch.sgml
index 382e54be91..08781778f8 100644
--- a/doc/src/sgml/fuzzystrmatch.sgml
+++ b/doc/src/sgml/fuzzystrmatch.sgml
@@ -241,4 +241,101 @@ test=# SELECT dmetaphone('gumbo');
</screen>
</sect2>
+ <sect2>
+ <title>Daitch-Mokotoff Soundex</title>
+
+ <para>
+ Compared to the American Soundex System implemented in the
+ <function>soundex</function> function, the major improvements of the
+ Daitch-Mokotoff Soundex System are:
+
+ <itemizedlist spacing="compact" mark="bullet">
+ <listitem>
+ <para>
+ Information is coded to the first six meaningful letters rather than
+ four.
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ The initial letter is coded rather than kept as is.
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ Where two consecutive letters have a single sound, they are coded as a
+ single number.
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ When a letter or combination of letters may have two different sounds,
+ it is double coded under the two different codes.
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ A letter or combination of letters maps into ten possible codes rather
+ than seven.
+ </para>
+ </listitem>
+ </itemizedlist>
+ </para>
+
+ <indexterm>
+ <primary>daitch_mokotoff</primary>
+ </indexterm>
+
+ <para>
+ The following function generates Daitch-Mokotoff soundex codes for matching
+ of similar-sounding input:
+ </para>
+
+<synopsis>
+daitch_mokotoff(text source) returns text
+</synopsis>
+
+ <para>
+ Since a Daitch-Mokotoff soundex code consists of only 6 digits,
+ <literal>source</literal> should be preferably a single word or name.
+ Any alternative soundex codes are separated by space, which makes the returned
+ text suited for use in Full Text Search, see <xref linkend="textsearch"/> and
+ <xref linkend="functions-textsearch"/>.
+ </para>
+
+ <para>
+ Example:
+ </para>
+
+<programlisting>
+CREATE OR REPLACE FUNCTION soundex_name(v_name text) RETURNS text AS $$
+ SELECT string_agg(daitch_mokotoff(n), ' ')
+ FROM regexp_split_to_table(v_name, '\s+') AS n
+$$ LANGUAGE sql STRICT IMMUTABLE PARALLEL SAFE;
+
+CREATE OR REPLACE FUNCTION soundex_tsvector(v_name text) RETURNS tsvector AS $$
+ SELECT to_tsvector('simple', soundex_name(v_name))
+$$ LANGUAGE sql STRICT IMMUTABLE PARALLEL SAFE;
+
+CREATE OR REPLACE FUNCTION soundex_tsquery(v_name text) RETURNS tsquery AS $$
+ SELECT to_tsquery('simple', quote_literal(soundex_name(v_name)))
+$$ LANGUAGE sql STRICT IMMUTABLE PARALLEL SAFE;
+
+-- Note that searches could be more efficient with the tsvector in a separate column
+-- (no recalculation on table row recheck).
+CREATE TABLE s (nm text);
+CREATE INDEX ix_s_txt ON s USING gin (soundex_tsvector(nm)) WITH (fastupdate = off);
+
+INSERT INTO s VALUES ('John Doe');
+INSERT INTO s VALUES ('Jane Roe');
+INSERT INTO s VALUES ('Public John Q.');
+INSERT INTO s VALUES ('George Best');
+
+SELECT * FROM s WHERE soundex_tsvector(nm) @@ soundex_tsquery('john');
+SELECT * FROM s WHERE soundex_tsvector(nm) @@ soundex_tsquery('jane doe');
+SELECT * FROM s WHERE soundex_tsvector(nm) @@ soundex_tsquery('john public');
+SELECT * FROM s WHERE soundex_tsvector(nm) @@ soundex_tsquery('besst, giorgio');
+</programlisting>
+ </sect2>
+
</sect1>
^ permalink raw reply [nested|flat] 19+ messages in thread
* Re: daitch_mokotoff module
2022-01-03 19:16 Re: daitch_mokotoff module Andres Freund <[email protected]>
2022-01-04 13:49 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
2022-01-04 20:18 ` Re: daitch_mokotoff module Thomas Munro <[email protected]>
2022-01-05 07:05 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
2022-01-05 20:08 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
2022-02-03 14:27 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
2022-12-07 18:56 ` Re: daitch_mokotoff module Andres Freund <[email protected]>
2022-12-21 09:26 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
@ 2022-12-22 12:00 ` Dag Lem <[email protected]>
2022-12-22 13:27 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
0 siblings, 1 reply; 19+ messages in thread
From: Dag Lem @ 2022-12-22 12:00 UTC (permalink / raw)
To: Andres Freund <[email protected]>; +Cc: PostgreSQL Hackers <[email protected]>; Thomas Munro <[email protected]>; Julien Rouhaud <[email protected]>
I noticed that the Meson builds failed in Cfbot, the updated patch adds
a missing "include_directories" line to meson.build.
Best regards
Dag Lem
Attachments:
[text/x-patch] v8-daitch_mokotoff.patch (39.1K, ../../[email protected]/2-v8-daitch_mokotoff.patch)
download | inline diff:
diff --git a/contrib/fuzzystrmatch/Makefile b/contrib/fuzzystrmatch/Makefile
index 0704894f88..12baf2d884 100644
--- a/contrib/fuzzystrmatch/Makefile
+++ b/contrib/fuzzystrmatch/Makefile
@@ -3,14 +3,15 @@
MODULE_big = fuzzystrmatch
OBJS = \
$(WIN32RES) \
+ daitch_mokotoff.o \
dmetaphone.o \
fuzzystrmatch.o
EXTENSION = fuzzystrmatch
-DATA = fuzzystrmatch--1.1.sql fuzzystrmatch--1.0--1.1.sql
+DATA = fuzzystrmatch--1.1.sql fuzzystrmatch--1.1--1.2.sql fuzzystrmatch--1.0--1.1.sql
PGFILEDESC = "fuzzystrmatch - similarities and distance between strings"
-REGRESS = fuzzystrmatch
+REGRESS = fuzzystrmatch fuzzystrmatch_utf8
ifdef USE_PGXS
PG_CONFIG = pg_config
@@ -22,3 +23,14 @@ top_builddir = ../..
include $(top_builddir)/src/Makefile.global
include $(top_srcdir)/contrib/contrib-global.mk
endif
+
+# Force this dependency to be known even without dependency info built:
+daitch_mokotoff.o: daitch_mokotoff.h
+
+daitch_mokotoff.h: daitch_mokotoff_header.pl
+ perl $< $@
+
+distprep: daitch_mokotoff.h
+
+maintainer-clean:
+ rm -f daitch_mokotoff.h
diff --git a/contrib/fuzzystrmatch/daitch_mokotoff.c b/contrib/fuzzystrmatch/daitch_mokotoff.c
new file mode 100644
index 0000000000..d4ad95c283
--- /dev/null
+++ b/contrib/fuzzystrmatch/daitch_mokotoff.c
@@ -0,0 +1,596 @@
+/*
+ * Daitch-Mokotoff Soundex
+ *
+ * Copyright (c) 2021 Finance Norway
+ * Author: Dag Lem <[email protected]>
+ *
+ * This implementation of the Daitch-Mokotoff Soundex System aims at high
+ * performance.
+ *
+ * - The processing of each phoneme is initiated by an O(1) table lookup.
+ * - For phonemes containing more than one character, a coding tree is traversed
+ * to process the complete phoneme.
+ * - The (alternate) soundex codes are produced digit by digit in-place in
+ * another tree structure.
+ *
+ * References:
+ *
+ * https://www.avotaynu.com/soundex.htm
+ * https://www.jewishgen.org/InfoFiles/Soundex.html
+ * https://familypedia.fandom.com/wiki/Daitch-Mokotoff_Soundex
+ * https://stevemorse.org/census/soundex.html (dmlat.php, dmsoundex.php)
+ * https://github.com/apache/commons-codec/ (dmrules.txt, DaitchMokotoffSoundex.java)
+ * https://metacpan.org/pod/Text::Phonetic (DaitchMokotoff.pm)
+ *
+ * A few notes on other implementations:
+ *
+ * - All other known implementations have the same unofficial rules for "UE",
+ * these are also adapted by this implementation (0, 1, NC).
+ * - The only other known implementation which is capable of generating all
+ * correct soundex codes in all cases is the JOS Soundex Calculator at
+ * https://www.jewishgen.org/jos/jossound.htm
+ * - "J" is considered (only) a vowel in dmlat.php
+ * - The official rules for "RS" are commented out in dmlat.php
+ * - Identical code digits for adjacent letters are not collapsed correctly in
+ * dmsoundex.php when double digit codes are involved. E.g. "BESST" yields
+ * 744300 instead of 743000 as for "BEST".
+ * - "J" is considered (only) a consonant in DaitchMokotoffSoundex.java
+ * - "Y" is not considered a vowel in DaitchMokotoffSoundex.java
+ *
+ * Permission to use, copy, modify, and distribute this software and its
+ * documentation for any purpose, without fee, and without a written agreement
+ * is hereby granted, provided that the above copyright notice and this
+ * paragraph and the following two paragraphs appear in all copies.
+ *
+ * IN NO EVENT SHALL THE AUTHOR OR DISTRIBUTORS BE LIABLE TO ANY PARTY FOR
+ * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING
+ * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS
+ * DOCUMENTATION, EVEN IF THE AUTHOR OR DISTRIBUTORS HAVE BEEN ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ * THE AUTHOR AND DISTRIBUTORS SPECIFICALLY DISCLAIM ANY WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
+ * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS
+ * ON AN "AS IS" BASIS, AND THE AUTHOR AND DISTRIBUTORS HAS NO OBLIGATIONS TO
+ * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
+*/
+
+#include "postgres.h"
+
+#include "lib/stringinfo.h"
+#include "mb/pg_wchar.h"
+#include "utils/builtins.h"
+#include "utils/memutils.h"
+
+
+/*
+ * The soundex coding chart table is adapted from from
+ * https://www.jewishgen.org/InfoFiles/Soundex.html
+ * See daitch_mokotoff_header.pl for details.
+*/
+
+#define DM_CODE_DIGITS 6
+
+/* Coding chart table: Soundex codes */
+typedef char dm_code[2 + 1]; /* One or two sequential code digits + NUL */
+typedef dm_code dm_codes[3]; /* Start of name, before a vowel, any other */
+
+/* Coding chart table: Letter in input sequence */
+struct dm_letter
+{
+ char letter; /* Present letter in sequence */
+ const struct dm_letter *letters; /* List of possible successive letters */
+ const dm_codes *codes; /* Code sequence(s) for complete sequence */
+};
+
+typedef struct dm_letter dm_letter;
+
+/* Generated coding chart table */
+#include "daitch_mokotoff.h"
+
+/* Node in soundex code tree */
+struct dm_node
+{
+ int soundex_length; /* Length of generated soundex code */
+ char soundex[DM_CODE_DIGITS + 1]; /* Soundex code */
+ int is_leaf; /* Candidate for complete soundex code */
+ int last_update; /* Letter number for last update of node */
+ char code_digit; /* Last code digit, 0 - 9 */
+
+ /*
+ * One or two alternate code digits leading to this node. If there are two
+ * digits, one of them is always an 'X'. Repeated code digits and 'X' lead
+ * back to the same node.
+ */
+ char prev_code_digits[2];
+ /* One or two alternate code digits moving forward. */
+ char next_code_digits[2];
+ /* ORed together code index(es) used to reach current node. */
+ int prev_code_index;
+ int next_code_index;
+ /* Nodes branching out from this node. */
+ struct dm_node *children[DM_MAX_ALTERNATE_CODES + 1];
+ /* Next node in linked list. Alternating index for each iteration. */
+ struct dm_node *next[2];
+};
+
+typedef struct dm_node dm_node;
+
+
+/* Internal C implementation */
+static int daitch_mokotoff_coding(char *word, StringInfo soundex);
+
+
+PG_FUNCTION_INFO_V1(daitch_mokotoff);
+
+Datum
+daitch_mokotoff(PG_FUNCTION_ARGS)
+{
+ text *arg = PG_GETARG_TEXT_PP(0);
+ char *string;
+ StringInfoData soundex;
+ text *retval;
+ MemoryContext old_ctx,
+ tmp_ctx;
+
+ tmp_ctx = AllocSetContextCreate(CurrentMemoryContext,
+ "daitch_mokotoff temporary context",
+ ALLOCSET_DEFAULT_SIZES);
+ old_ctx = MemoryContextSwitchTo(tmp_ctx);
+
+ string = pg_server_to_any(text_to_cstring(arg), VARSIZE_ANY_EXHDR(arg), PG_UTF8);
+ initStringInfo(&soundex);
+
+ if (!daitch_mokotoff_coding(string, &soundex))
+ {
+ /* No encodable characters in input. */
+ MemoryContextSwitchTo(old_ctx);
+ MemoryContextDelete(tmp_ctx);
+ PG_RETURN_NULL();
+ }
+
+ string = pg_any_to_server(soundex.data, soundex.len, PG_UTF8);
+ MemoryContextSwitchTo(old_ctx);
+ retval = cstring_to_text(string);
+ MemoryContextDelete(tmp_ctx);
+
+ PG_RETURN_TEXT_P(retval);
+}
+
+
+/* Template for new node in soundex code tree. */
+static const dm_node start_node = {
+ .soundex_length = 0,
+ .soundex = "000000 ", /* Six digits + joining space */
+ .is_leaf = 0,
+ .last_update = 0,
+ .code_digit = '\0',
+ .prev_code_digits = {'\0', '\0'},
+ .next_code_digits = {'\0', '\0'},
+ .prev_code_index = 0,
+ .next_code_index = 0,
+ .children = {NULL},
+ .next = {NULL}
+};
+
+/* Dummy soundex codes at end of input. */
+static const dm_codes end_codes[2] =
+{
+ {
+ "X", "X", "X"
+ }
+};
+
+
+/* Initialize soundex code tree node for next code digit. */
+static void
+initialize_node(dm_node * node, int last_update)
+{
+ if (node->last_update < last_update)
+ {
+ node->prev_code_digits[0] = node->next_code_digits[0];
+ node->prev_code_digits[1] = node->next_code_digits[1];
+ node->next_code_digits[0] = '\0';
+ node->next_code_digits[1] = '\0';
+ node->prev_code_index = node->next_code_index;
+ node->next_code_index = 0;
+ node->is_leaf = 0;
+ node->last_update = last_update;
+ }
+}
+
+
+/* Update soundex code tree node with next code digit. */
+static void
+add_next_code_digit(dm_node * node, int code_index, char code_digit)
+{
+ /* OR in index 1 or 2. */
+ node->next_code_index |= code_index;
+
+ if (!node->next_code_digits[0])
+ {
+ node->next_code_digits[0] = code_digit;
+ }
+ else if (node->next_code_digits[0] != code_digit)
+ {
+ node->next_code_digits[1] = code_digit;
+ }
+}
+
+
+/* Mark soundex code tree node as leaf. */
+static void
+set_leaf(dm_node * first_node[2], dm_node * last_node[2], dm_node * node, int ix_node)
+{
+ if (!node->is_leaf)
+ {
+ node->is_leaf = 1;
+
+ if (first_node[ix_node] == NULL)
+ {
+ first_node[ix_node] = node;
+ }
+ else
+ {
+ last_node[ix_node]->next[ix_node] = node;
+ }
+
+ last_node[ix_node] = node;
+ node->next[ix_node] = NULL;
+ }
+}
+
+
+/* Find next node corresponding to code digit, or create a new node. */
+static dm_node *
+find_or_create_child_node(dm_node * parent, char code_digit, StringInfo soundex)
+{
+ dm_node **nodes;
+ dm_node *node;
+ int i;
+
+ for (nodes = parent->children, i = 0; (node = nodes[i]); i++)
+ {
+ if (node->code_digit == code_digit)
+ {
+ /* Found existing child node. Skip completed nodes. */
+ return node->soundex_length < DM_CODE_DIGITS ? node : NULL;
+ }
+ }
+
+ /* Create new child node. */
+ Assert(i < DM_MAX_ALTERNATE_CODES);
+ node = palloc(sizeof(dm_node));
+ nodes[i] = node;
+
+ *node = start_node;
+ memcpy(node->soundex, parent->soundex, sizeof(parent->soundex));
+ node->soundex_length = parent->soundex_length;
+ node->soundex[node->soundex_length++] = code_digit;
+ node->code_digit = code_digit;
+ node->next_code_index = node->prev_code_index;
+
+ if (node->soundex_length < DM_CODE_DIGITS)
+ {
+ return node;
+ }
+ else
+ {
+ /* Append completed soundex code to soundex string. */
+ appendBinaryStringInfoNT(soundex, node->soundex, DM_CODE_DIGITS + 1);
+ return NULL;
+ }
+}
+
+
+/* Update node for next code digit(s). */
+static void
+update_node(dm_node * first_node[2], dm_node * last_node[2], dm_node * node, int ix_node,
+ int letter_no, int prev_code_index, int next_code_index,
+ const char *next_code_digits, int digit_no,
+ StringInfo soundex)
+{
+ int i;
+ char next_code_digit = next_code_digits[digit_no];
+ int num_dirty_nodes = 0;
+ dm_node *dirty_nodes[2];
+
+ initialize_node(node, letter_no);
+
+ if (node->prev_code_index && !(node->prev_code_index & prev_code_index))
+ {
+ /*
+ * If the sound (vowel / consonant) of this letter encoding doesn't
+ * correspond to the coding index of the previous letter, we skip this
+ * letter encoding. Note that currently, only "J" can be either a
+ * vowel or a consonant.
+ */
+ return;
+ }
+
+ if (next_code_digit == 'X' ||
+ (digit_no == 0 &&
+ (node->prev_code_digits[0] == next_code_digit ||
+ node->prev_code_digits[1] == next_code_digit)))
+ {
+ /* The code digit is the same as one of the previous (i.e. not added). */
+ dirty_nodes[num_dirty_nodes++] = node;
+ }
+
+ if (next_code_digit != 'X' &&
+ (digit_no > 0 ||
+ node->prev_code_digits[0] != next_code_digit ||
+ node->prev_code_digits[1]))
+ {
+ /* The code digit is different from one of the previous (i.e. added). */
+ node = find_or_create_child_node(node, next_code_digit, soundex);
+ if (node)
+ {
+ initialize_node(node, letter_no);
+ dirty_nodes[num_dirty_nodes++] = node;
+ }
+ }
+
+ for (i = 0; i < num_dirty_nodes; i++)
+ {
+ /* Add code digit leading to the current node. */
+ add_next_code_digit(dirty_nodes[i], next_code_index, next_code_digit);
+
+ if (next_code_digits[++digit_no])
+ {
+ update_node(first_node, last_node, dirty_nodes[i], ix_node,
+ letter_no, prev_code_index, next_code_index,
+ next_code_digits, digit_no,
+ soundex);
+ }
+ else
+ {
+ /* Add incomplete leaf node to linked list. */
+ set_leaf(first_node, last_node, dirty_nodes[i], ix_node);
+ }
+ }
+}
+
+
+/* Update soundex tree leaf nodes. */
+static void
+update_leaves(dm_node * first_node[2], int *ix_node, int letter_no,
+ const dm_codes * codes, const dm_codes * next_codes,
+ StringInfo soundex)
+{
+ int i,
+ j,
+ code_index;
+ dm_node *node,
+ *last_node[2];
+ const dm_code *code,
+ *next_code;
+ int ix_node_next = (*ix_node + 1) & 1; /* Alternating index: 0, 1 */
+
+ /* Initialize for new linked list of leaves. */
+ first_node[ix_node_next] = NULL;
+ last_node[ix_node_next] = NULL;
+
+ /* Process all nodes. */
+ for (node = first_node[*ix_node]; node; node = node->next[*ix_node])
+ {
+ /* One or two alternate code sequences. */
+ for (i = 0; i < 2 && (code = codes[i]) && code[0][0]; i++)
+ {
+ /* Coding for previous letter - before vowel: 1, all other: 2 */
+ int prev_code_index = (code[0][0] > '1') + 1;
+
+ /* One or two alternate next code sequences. */
+ for (j = 0; j < 2 && (next_code = next_codes[j]) && next_code[0][0]; j++)
+ {
+ /* Determine which code to use. */
+ if (letter_no == 0)
+ {
+ /* This is the first letter. */
+ code_index = 0;
+ }
+ else if (next_code[0][0] <= '1')
+ {
+ /* The next letter is a vowel. */
+ code_index = 1;
+ }
+ else
+ {
+ /* All other cases. */
+ code_index = 2;
+ }
+
+ /* One or two sequential code digits. */
+ update_node(first_node, last_node, node, ix_node_next,
+ letter_no, prev_code_index, code_index,
+ code[code_index], 0,
+ soundex);
+ }
+ }
+ }
+
+ *ix_node = ix_node_next;
+}
+
+
+/* Mapping from ISO8859-1 to upper-case ASCII */
+static const char iso8859_1_to_ascii_upper[] =
+/*
+"`abcdefghijklmnopqrstuvwxyz{|}~ ¡¢£¤¥¦§¨©ª«¬ ®¯°±²³´µ¶·¸¹º»¼½¾¿ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖ×ØÙÚÛÜÝÞßàáâãäåæçèéêëìíîïðñòóôõö÷øùúûüýþÿ"
+*/
+"`ABCDEFGHIJKLMNOPQRSTUVWXYZ{|}~ ! ?AAAAAAECEEEEIIIIDNOOOOO*OUUUUYDSAAAAAAECEEEEIIIIDNOOOOO/OUUUUYDY";
+
+
+/* Return next character, converted from UTF-8 to uppercase ASCII. */
+static char
+read_char(unsigned char *str, int *ix)
+{
+ /* Substitute character for skipped code points. */
+ const char na = '\x1a';
+ pg_wchar c;
+
+ /* Decode UTF-8 character to ISO 10646 code point. */
+ str += *ix;
+ c = utf8_to_unicode(str);
+ *ix += pg_utf_mblen(str);
+
+ if (c >= (unsigned char) '[' && c <= (unsigned char) ']')
+ {
+ /* ASCII characters [, \, and ] are reserved for Ą, Ę, and Ţ/Ț. */
+ return na;
+ }
+ else if (c < 0x60)
+ {
+ /* Non-lowercase ASCII character. */
+ return c;
+ }
+ else if (c < 0x100)
+ {
+ /* ISO-8859-1 code point, converted to upper-case ASCII. */
+ return iso8859_1_to_ascii_upper[c - 0x60];
+ }
+ else
+ {
+ /* Conversion of non-ASCII characters in the coding chart. */
+ switch (c)
+ {
+ case 0x0104:
+ case 0x0105:
+ /* Ą/ą */
+ return '[';
+ case 0x0118:
+ case 0x0119:
+ /* Ę/ę */
+ return '\\';
+ case 0x0162:
+ case 0x0163:
+ case 0x021A:
+ case 0x021B:
+ /* Ţ/ţ or Ț/ț */
+ return ']';
+ default:
+ return na;
+ }
+ }
+}
+
+
+/* Read next ASCII character, skipping any characters not in [A-\]]. */
+static char
+read_valid_char(char *str, int *ix)
+{
+ char c;
+
+ while ((c = read_char((unsigned char *) str, ix)))
+ {
+ if (c >= 'A' && c <= ']')
+ {
+ break;
+ }
+ }
+
+ return c;
+}
+
+
+/* Return sound coding for "letter" (letter sequence) */
+static const dm_codes *
+read_letter(char *str, int *ix)
+{
+ char c,
+ cmp;
+ int i,
+ j;
+ const dm_letter *letters;
+ const dm_codes *codes;
+
+ /* First letter in sequence. */
+ if (!(c = read_valid_char(str, ix)))
+ {
+ return NULL;
+ }
+ letters = &letter_[c - 'A'];
+ codes = letters->codes;
+ i = *ix;
+
+ /* Any subsequent letters in sequence. */
+ while ((letters = letters->letters) && (c = read_valid_char(str, &i)))
+ {
+ for (j = 0; (cmp = letters[j].letter); j++)
+ {
+ if (cmp == c)
+ {
+ /* Letter found. */
+ letters = &letters[j];
+ if (letters->codes)
+ {
+ /* Coding for letter sequence found. */
+ codes = letters->codes;
+ *ix = i;
+ }
+ break;
+ }
+ }
+ if (!cmp)
+ {
+ /* The sequence of letters has no coding. */
+ break;
+ }
+ }
+
+ return codes;
+}
+
+
+/* Generate all Daitch-Mokotoff soundex codes for word, separated by space. */
+static int
+daitch_mokotoff_coding(char *word, StringInfo soundex)
+{
+ int i = 0;
+ int letter_no = 0;
+ int ix_node = 0;
+ const dm_codes *codes,
+ *next_codes;
+ dm_node *first_node[2],
+ *node;
+
+ /* First letter. */
+ if (!(codes = read_letter(word, &i)))
+ {
+ /* No encodable character in input. */
+ return 0;
+ }
+
+ /* Starting point. */
+ first_node[ix_node] = palloc(sizeof(dm_node));
+ *first_node[ix_node] = start_node;
+
+ /*
+ * Loop until either the word input is exhausted, or all generated soundex
+ * codes are completed to six digits.
+ */
+ while (codes && first_node[ix_node])
+ {
+ next_codes = read_letter(word, &i);
+
+ /* Update leaf nodes. */
+ update_leaves(first_node, &ix_node, letter_no,
+ codes, next_codes ? next_codes : end_codes,
+ soundex);
+
+ codes = next_codes;
+ letter_no++;
+ }
+
+ /* Append all remaining (incomplete) soundex codes. */
+ for (node = first_node[ix_node]; node; node = node->next[ix_node])
+ {
+ appendBinaryStringInfoNT(soundex, node->soundex, DM_CODE_DIGITS + 1);
+ }
+
+ /* Terminate string at the final space. */
+ soundex->len--;
+ soundex->data[soundex->len] = '\0';
+
+ return 1;
+}
diff --git a/contrib/fuzzystrmatch/daitch_mokotoff_header.pl b/contrib/fuzzystrmatch/daitch_mokotoff_header.pl
new file mode 100755
index 0000000000..807b5fb8c5
--- /dev/null
+++ b/contrib/fuzzystrmatch/daitch_mokotoff_header.pl
@@ -0,0 +1,260 @@
+#!/bin/perl
+#
+# Generation of types and lookup tables for Daitch-Mokotoff soundex.
+#
+# Copyright (c) 2021 Finance Norway
+# Author: Dag Lem <[email protected]>
+#
+# Permission to use, copy, modify, and distribute this software and its
+# documentation for any purpose, without fee, and without a written agreement
+# is hereby granted, provided that the above copyright notice and this
+# paragraph and the following two paragraphs appear in all copies.
+#
+# IN NO EVENT SHALL THE AUTHOR OR DISTRIBUTORS BE LIABLE TO ANY PARTY FOR
+# DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING
+# LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS
+# DOCUMENTATION, EVEN IF THE AUTHOR OR DISTRIBUTORS HAVE BEEN ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+#
+# THE AUTHOR AND DISTRIBUTORS SPECIFICALLY DISCLAIM ANY WARRANTIES,
+# INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
+# AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS
+# ON AN "AS IS" BASIS, AND THE AUTHOR AND DISTRIBUTORS HAS NO OBLIGATIONS TO
+# PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
+#
+
+use strict;
+use warnings;
+use utf8;
+use open IO => ':utf8', ':std';
+use Data::Dumper;
+
+die "Usage: $0 OUTPUT_FILE\n" if @ARGV != 1;
+my $output_file = $ARGV[0];
+
+# Open the output file
+open my $OUTPUT, '>', $output_file
+ or die "Could not open output file $output_file: $!\n";
+
+# Parse code table and generate tree for letter transitions.
+my %codes;
+my %alternates;
+my $table = [{}, [["","",""]]];
+while (<DATA>) {
+ chomp;
+ my ($letters, $codes) = split(/\s+/);
+ my @codes = map { [ split(/,/) ] } split(/\|/, $codes);
+
+ # Find alternate code transitions for calculation of storage.
+ # The first character ("start of a name") can never yield more than two alternate codes,
+ # and is not considered here.
+ if (@codes > 1) {
+ for my $j (1..2) { # Codes for "before a vowel" and "any other"
+ for my $i (0..1) { # Alternate codes
+ # Identical code digits for adjacent letters are collapsed.
+ # For each possible non-transition due to code digit
+ # collapsing, find all alternate transitions.
+ my ($present, $next) = ($codes[$i][$j], $codes[($i + 1)%2][$j]);
+ next if length($present) != 1;
+ $next = $present ne substr($next, 0, 1) ? substr($next, 0, 1) : substr($next, -1, 1);
+ $alternates{$present}{$next} = 1;
+ }
+ }
+ }
+ my $key = "codes_" . join("_or_", map { join("_", @$_) } @codes);
+ my $val = join(",\n", map { "\t{\n\t\t" . join(", ", map { "\"$_\"" } @$_) . "\n\t}" } @codes);
+ $codes{$key} = $val;
+
+ for my $letter (split(/,/, $letters)) {
+ my $ref = $table->[0];
+ # Link each character to the next in the letter combination.
+ my @c = split(//, $letter);
+ my $last_c = pop(@c);
+ for my $c (@c) {
+ $ref->{$c} //= [ {}, undef ];
+ $ref->{$c}[0] //= {};
+ $ref = $ref->{$c}[0];
+ }
+ # The sound code for the letter combination is stored at the last character.
+ $ref->{$last_c}[1] = $key;
+ }
+}
+close(DATA);
+
+# Add alternates by following transitions to 'X' (not coded).
+my $alt_x = $alternates{"X"};
+delete $alt_x->{"X"};
+while (my ($k, $v) = each %alternates) {
+ if (delete $v->{"X"}) {
+ for my $x (keys %$alt_x) {
+ $v->{$x} = 1;
+ }
+ }
+}
+
+# Find the maximum number of alternate codes in one position.
+# Add two for any additional final code digit transitions.
+my $max_alt = (sort { $b <=> $a } (map { scalar keys %$_ } values %alternates))[0] + 2;
+
+print $OUTPUT <<EOF;
+/*
+ * Constants and lookup tables for Daitch-Mokotoff Soundex
+ *
+ * Copyright (c) 2021 Finance Norway
+ * Author: Dag Lem <dag\@nimrod.no>
+ *
+ * This file is generated by daitch_mokotoff_header.pl
+ *
+ * Permission to use, copy, modify, and distribute this software and its
+ * documentation for any purpose, without fee, and without a written agreement
+ * is hereby granted, provided that the above copyright notice and this
+ * paragraph and the following two paragraphs appear in all copies.
+ *
+ * IN NO EVENT SHALL THE AUTHOR OR DISTRIBUTORS BE LIABLE TO ANY PARTY FOR
+ * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING
+ * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS
+ * DOCUMENTATION, EVEN IF THE AUTHOR OR DISTRIBUTORS HAVE BEEN ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ * THE AUTHOR AND DISTRIBUTORS SPECIFICALLY DISCLAIM ANY WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
+ * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS
+ * ON AN "AS IS" BASIS, AND THE AUTHOR AND DISTRIBUTORS HAS NO OBLIGATIONS TO
+ * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
+ */
+
+#define DM_MAX_ALTERNATE_CODES $max_alt
+
+/* Codes for letter sequence at start of name, before a vowel, and any other. */
+EOF
+
+for my $key (sort keys %codes) {
+ print $OUTPUT "static const dm_codes $key\[2\] =\n{\n" . $codes{$key} . "\n};\n";
+}
+
+print $OUTPUT <<EOF;
+
+/* Coding for alternative following letters in sequence. */
+EOF
+
+sub hash2code {
+ my ($ref, $letter) = @_;
+
+ my @letters = ();
+
+ my $h = $ref->[0];
+ for my $key (sort keys %$h) {
+ $ref = $h->{$key};
+ my $children = "NULL";
+ if (defined $ref->[0]) {
+ $children = "letter_$letter$key";
+ hash2code($ref, "$letter$key");
+ }
+ my $codes = $ref->[1] // "NULL";
+ push(@letters, "\t{\n\t\t'$key', $children, $codes\n\t}");
+ }
+
+ print $OUTPUT "static const dm_letter letter_$letter\[\] =\n{\n";
+ for (@letters) {
+ print $OUTPUT "$_,\n";
+ }
+ print $OUTPUT "\t{\n\t\t'\\0'\n\t}\n";
+ print $OUTPUT "};\n";
+}
+
+hash2code($table, '');
+
+close $OUTPUT;
+
+# Table adapted from https://www.jewishgen.org/InfoFiles/Soundex.html
+#
+# The conversion from the coding chart to the table should be self
+# explanatory, but note the differences stated below.
+#
+# X = NC (not coded)
+#
+# The non-ASCII letters in the coding chart are coded with substitute
+# lowercase ASCII letters, which sort after the uppercase ASCII letters:
+#
+# Ą => a (use '[' for table lookup)
+# Ę => e (use '\\' for table lookup)
+# Ţ => t (use ']' for table lookup)
+#
+# The rule for "UE" does not correspond to the coding chart, however
+# it is used by all other known implementations, including the one at
+# https://www.jewishgen.org/jos/jossound.htm (try e.g. "bouey").
+#
+# Note that the implementation assumes that vowels are assigned code
+# 0 or 1. "J" can be either a vowel or a consonant.
+#
+
+__DATA__
+AI,AJ,AY 0,1,X
+AU 0,7,X
+a X,X,6|X,X,X
+A 0,X,X
+B 7,7,7
+CHS 5,54,54
+CH 5,5,5|4,4,4
+CK 5,5,5|45,45,45
+CZ,CS,CSZ,CZS 4,4,4
+C 5,5,5|4,4,4
+DRZ,DRS 4,4,4
+DS,DSH,DSZ 4,4,4
+DZ,DZH,DZS 4,4,4
+D,DT 3,3,3
+EI,EJ,EY 0,1,X
+EU 1,1,X
+e X,X,6|X,X,X
+E 0,X,X
+FB 7,7,7
+F 7,7,7
+G 5,5,5
+H 5,5,X
+IA,IE,IO,IU 1,X,X
+I 0,X,X
+J 1,X,X|4,4,4
+KS 5,54,54
+KH 5,5,5
+K 5,5,5
+L 8,8,8
+MN 66,66,66
+M 6,6,6
+NM 66,66,66
+N 6,6,6
+OI,OJ,OY 0,1,X
+O 0,X,X
+P,PF,PH 7,7,7
+Q 5,5,5
+RZ,RS 94,94,94|4,4,4
+R 9,9,9
+SCHTSCH,SCHTSH,SCHTCH 2,4,4
+SCH 4,4,4
+SHTCH,SHCH,SHTSH 2,4,4
+SHT,SCHT,SCHD 2,43,43
+SH 4,4,4
+STCH,STSCH,SC 2,4,4
+STRZ,STRS,STSH 2,4,4
+ST 2,43,43
+SZCZ,SZCS 2,4,4
+SZT,SHD,SZD,SD 2,43,43
+SZ 4,4,4
+S 4,4,4
+TCH,TTCH,TTSCH 4,4,4
+TH 3,3,3
+TRZ,TRS 4,4,4
+TSCH,TSH 4,4,4
+TS,TTS,TTSZ,TC 4,4,4
+TZ,TTZ,TZS,TSZ 4,4,4
+t 3,3,3|4,4,4
+T 3,3,3
+UI,UJ,UY,UE 0,1,X
+U 0,X,X
+V 7,7,7
+W 7,7,7
+X 5,54,54
+Y 1,X,X
+ZDZ,ZDZH,ZHDZH 2,4,4
+ZD,ZHD 2,43,43
+ZH,ZS,ZSCH,ZSH 4,4,4
+Z 4,4,4
diff --git a/contrib/fuzzystrmatch/expected/fuzzystrmatch.out b/contrib/fuzzystrmatch/expected/fuzzystrmatch.out
index 493c95cdfa..f62ddad4ee 100644
--- a/contrib/fuzzystrmatch/expected/fuzzystrmatch.out
+++ b/contrib/fuzzystrmatch/expected/fuzzystrmatch.out
@@ -65,3 +65,174 @@ SELECT dmetaphone_alt('gumbo');
KMP
(1 row)
+-- Wovels
+SELECT daitch_mokotoff('Augsburg');
+ daitch_mokotoff
+-----------------
+ 054795
+(1 row)
+
+SELECT daitch_mokotoff('Breuer');
+ daitch_mokotoff
+-----------------
+ 791900
+(1 row)
+
+SELECT daitch_mokotoff('Freud');
+ daitch_mokotoff
+-----------------
+ 793000
+(1 row)
+
+-- The letter "H"
+SELECT daitch_mokotoff('Halberstadt');
+ daitch_mokotoff
+-----------------
+ 587943 587433
+(1 row)
+
+SELECT daitch_mokotoff('Mannheim');
+ daitch_mokotoff
+-----------------
+ 665600
+(1 row)
+
+-- Adjacent sounds
+SELECT daitch_mokotoff('Chernowitz');
+ daitch_mokotoff
+-----------------
+ 596740 496740
+(1 row)
+
+-- Adjacent letters with identical adjacent code digits
+SELECT daitch_mokotoff('Cherkassy');
+ daitch_mokotoff
+-----------------
+ 595400 495400
+(1 row)
+
+SELECT daitch_mokotoff('Kleinman');
+ daitch_mokotoff
+-----------------
+ 586660
+(1 row)
+
+-- More than one word
+SELECT daitch_mokotoff('Nowy Targ');
+ daitch_mokotoff
+-----------------
+ 673950
+(1 row)
+
+-- Padded with "0"
+SELECT daitch_mokotoff('Berlin');
+ daitch_mokotoff
+-----------------
+ 798600
+(1 row)
+
+-- Other examples from https://www.avotaynu.com/soundex.htm
+SELECT daitch_mokotoff('Ceniow');
+ daitch_mokotoff
+-----------------
+ 567000 467000
+(1 row)
+
+SELECT daitch_mokotoff('Tsenyuv');
+ daitch_mokotoff
+-----------------
+ 467000
+(1 row)
+
+SELECT daitch_mokotoff('Holubica');
+ daitch_mokotoff
+-----------------
+ 587500 587400
+(1 row)
+
+SELECT daitch_mokotoff('Golubitsa');
+ daitch_mokotoff
+-----------------
+ 587400
+(1 row)
+
+SELECT daitch_mokotoff('Przemysl');
+ daitch_mokotoff
+-----------------
+ 794648 746480
+(1 row)
+
+SELECT daitch_mokotoff('Pshemeshil');
+ daitch_mokotoff
+-----------------
+ 746480
+(1 row)
+
+SELECT daitch_mokotoff('Rosochowaciec');
+ daitch_mokotoff
+---------------------------------------------------------
+ 945755 945754 945745 945744 944755 944754 944745 944744
+(1 row)
+
+SELECT daitch_mokotoff('Rosokhovatsets');
+ daitch_mokotoff
+-----------------
+ 945744
+(1 row)
+
+-- Ignored characters
+SELECT daitch_mokotoff('''OBrien');
+ daitch_mokotoff
+-----------------
+ 079600
+(1 row)
+
+SELECT daitch_mokotoff('O''Brien');
+ daitch_mokotoff
+-----------------
+ 079600
+(1 row)
+
+-- "Difficult" cases, likely to cause trouble for other implementations.
+SELECT daitch_mokotoff('CJC');
+ daitch_mokotoff
+-------------------------------------------
+ 550000 540000 545000 450000 400000 440000
+(1 row)
+
+SELECT daitch_mokotoff('BESST');
+ daitch_mokotoff
+-----------------
+ 743000
+(1 row)
+
+SELECT daitch_mokotoff('BOUEY');
+ daitch_mokotoff
+-----------------
+ 710000
+(1 row)
+
+SELECT daitch_mokotoff('HANNMANN');
+ daitch_mokotoff
+-----------------
+ 566600
+(1 row)
+
+SELECT daitch_mokotoff('MCCOYJR');
+ daitch_mokotoff
+---------------------------------------------------------
+ 651900 654900 654190 654490 645190 645490 641900 644900
+(1 row)
+
+SELECT daitch_mokotoff('ACCURSO');
+ daitch_mokotoff
+---------------------------------------------------------
+ 059400 054000 054940 054400 045940 045400 049400 044000
+(1 row)
+
+SELECT daitch_mokotoff('BIERSCHBACH');
+ daitch_mokotoff
+---------------------------------------------------------
+ 794575 794574 794750 794740 745750 745740 747500 747400
+(1 row)
+
diff --git a/contrib/fuzzystrmatch/expected/fuzzystrmatch_utf8.out b/contrib/fuzzystrmatch/expected/fuzzystrmatch_utf8.out
new file mode 100644
index 0000000000..32d8260383
--- /dev/null
+++ b/contrib/fuzzystrmatch/expected/fuzzystrmatch_utf8.out
@@ -0,0 +1,61 @@
+/*
+ * This test must be run in a database with UTF-8 encoding,
+ * because other encodings don't support all the characters used.
+ */
+SELECT getdatabaseencoding() <> 'UTF8'
+ AS skip_test \gset
+\if :skip_test
+\quit
+\endif
+set client_encoding = utf8;
+-- CREATE EXTENSION IF NOT EXISTS fuzzystrmatch;
+-- Accents
+SELECT daitch_mokotoff('Müller');
+ daitch_mokotoff
+-----------------
+ 689000
+(1 row)
+
+SELECT daitch_mokotoff('Schäfer');
+ daitch_mokotoff
+-----------------
+ 479000
+(1 row)
+
+SELECT daitch_mokotoff('Straßburg');
+ daitch_mokotoff
+-----------------
+ 294795
+(1 row)
+
+SELECT daitch_mokotoff('Éregon');
+ daitch_mokotoff
+-----------------
+ 095600
+(1 row)
+
+-- Special characters added at https://www.jewishgen.org/InfoFiles/Soundex.html
+SELECT daitch_mokotoff('gąszczu');
+ daitch_mokotoff
+-----------------
+ 564000 540000
+(1 row)
+
+SELECT daitch_mokotoff('brzęczy');
+ daitch_mokotoff
+-----------------------------
+ 794640 794400 746400 744000
+(1 row)
+
+SELECT daitch_mokotoff('ţamas');
+ daitch_mokotoff
+-----------------
+ 364000 464000
+(1 row)
+
+SELECT daitch_mokotoff('țamas');
+ daitch_mokotoff
+-----------------
+ 364000 464000
+(1 row)
+
diff --git a/contrib/fuzzystrmatch/expected/fuzzystrmatch_utf8_1.out b/contrib/fuzzystrmatch/expected/fuzzystrmatch_utf8_1.out
new file mode 100644
index 0000000000..37aead89c0
--- /dev/null
+++ b/contrib/fuzzystrmatch/expected/fuzzystrmatch_utf8_1.out
@@ -0,0 +1,8 @@
+/*
+ * This test must be run in a database with UTF-8 encoding,
+ * because other encodings don't support all the characters used.
+ */
+SELECT getdatabaseencoding() <> 'UTF8'
+ AS skip_test \gset
+\if :skip_test
+\quit
diff --git a/contrib/fuzzystrmatch/fuzzystrmatch--1.1--1.2.sql b/contrib/fuzzystrmatch/fuzzystrmatch--1.1--1.2.sql
new file mode 100644
index 0000000000..b9d7b229a3
--- /dev/null
+++ b/contrib/fuzzystrmatch/fuzzystrmatch--1.1--1.2.sql
@@ -0,0 +1,8 @@
+/* contrib/fuzzystrmatch/fuzzystrmatch--1.1--1.2.sql */
+
+-- complain if script is sourced in psql, rather than via ALTER EXTENSION
+\echo Use "ALTER EXTENSION fuzzystrmatch UPDATE TO '1.2'" to load this file. \quit
+
+CREATE FUNCTION daitch_mokotoff(text) RETURNS text
+AS 'MODULE_PATHNAME', 'daitch_mokotoff'
+LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
diff --git a/contrib/fuzzystrmatch/fuzzystrmatch.control b/contrib/fuzzystrmatch/fuzzystrmatch.control
index 3cd6660bf9..8b6e9fd993 100644
--- a/contrib/fuzzystrmatch/fuzzystrmatch.control
+++ b/contrib/fuzzystrmatch/fuzzystrmatch.control
@@ -1,6 +1,6 @@
# fuzzystrmatch extension
comment = 'determine similarities and distance between strings'
-default_version = '1.1'
+default_version = '1.2'
module_pathname = '$libdir/fuzzystrmatch'
relocatable = true
trusted = true
diff --git a/contrib/fuzzystrmatch/meson.build b/contrib/fuzzystrmatch/meson.build
index e6d06149ce..73178794c2 100644
--- a/contrib/fuzzystrmatch/meson.build
+++ b/contrib/fuzzystrmatch/meson.build
@@ -1,7 +1,16 @@
fuzzystrmatch_sources = files(
- 'fuzzystrmatch.c',
+ 'daitch_mokotoff.c',
'dmetaphone.c',
+ 'fuzzystrmatch.c',
+)
+
+daitch_mokotoff_h = custom_target('daitch_mokotoff',
+ input: 'daitch_mokotoff_header.pl',
+ output: 'daitch_mokotoff.h',
+ command: [perl, '@INPUT@', '@OUTPUT@'],
)
+generated_sources += daitch_mokotoff_h
+fuzzystrmatch_sources += daitch_mokotoff_h
if host_system == 'windows'
fuzzystrmatch_sources += rc_lib_gen.process(win32ver_rc, extra_args: [
@@ -11,6 +20,7 @@ endif
fuzzystrmatch = shared_module('fuzzystrmatch',
fuzzystrmatch_sources,
+ include_directories: include_directories('.'),
kwargs: contrib_mod_args,
)
contrib_targets += fuzzystrmatch
@@ -19,6 +29,7 @@ install_data(
'fuzzystrmatch.control',
'fuzzystrmatch--1.0--1.1.sql',
'fuzzystrmatch--1.1.sql',
+ 'fuzzystrmatch--1.1--1.2.sql',
kwargs: contrib_data_args,
)
@@ -29,6 +40,7 @@ tests += {
'regress': {
'sql': [
'fuzzystrmatch',
+ 'fuzzystrmatch_utf8',
],
},
}
diff --git a/contrib/fuzzystrmatch/sql/fuzzystrmatch.sql b/contrib/fuzzystrmatch/sql/fuzzystrmatch.sql
index f05dc28ffb..db05c7d6b6 100644
--- a/contrib/fuzzystrmatch/sql/fuzzystrmatch.sql
+++ b/contrib/fuzzystrmatch/sql/fuzzystrmatch.sql
@@ -19,3 +19,48 @@ SELECT metaphone('GUMBO', 4);
SELECT dmetaphone('gumbo');
SELECT dmetaphone_alt('gumbo');
+
+-- Wovels
+SELECT daitch_mokotoff('Augsburg');
+SELECT daitch_mokotoff('Breuer');
+SELECT daitch_mokotoff('Freud');
+
+-- The letter "H"
+SELECT daitch_mokotoff('Halberstadt');
+SELECT daitch_mokotoff('Mannheim');
+
+-- Adjacent sounds
+SELECT daitch_mokotoff('Chernowitz');
+
+-- Adjacent letters with identical adjacent code digits
+SELECT daitch_mokotoff('Cherkassy');
+SELECT daitch_mokotoff('Kleinman');
+
+-- More than one word
+SELECT daitch_mokotoff('Nowy Targ');
+
+-- Padded with "0"
+SELECT daitch_mokotoff('Berlin');
+
+-- Other examples from https://www.avotaynu.com/soundex.htm
+SELECT daitch_mokotoff('Ceniow');
+SELECT daitch_mokotoff('Tsenyuv');
+SELECT daitch_mokotoff('Holubica');
+SELECT daitch_mokotoff('Golubitsa');
+SELECT daitch_mokotoff('Przemysl');
+SELECT daitch_mokotoff('Pshemeshil');
+SELECT daitch_mokotoff('Rosochowaciec');
+SELECT daitch_mokotoff('Rosokhovatsets');
+
+-- Ignored characters
+SELECT daitch_mokotoff('''OBrien');
+SELECT daitch_mokotoff('O''Brien');
+
+-- "Difficult" cases, likely to cause trouble for other implementations.
+SELECT daitch_mokotoff('CJC');
+SELECT daitch_mokotoff('BESST');
+SELECT daitch_mokotoff('BOUEY');
+SELECT daitch_mokotoff('HANNMANN');
+SELECT daitch_mokotoff('MCCOYJR');
+SELECT daitch_mokotoff('ACCURSO');
+SELECT daitch_mokotoff('BIERSCHBACH');
diff --git a/contrib/fuzzystrmatch/sql/fuzzystrmatch_utf8.sql b/contrib/fuzzystrmatch/sql/fuzzystrmatch_utf8.sql
new file mode 100644
index 0000000000..f42c01a1bb
--- /dev/null
+++ b/contrib/fuzzystrmatch/sql/fuzzystrmatch_utf8.sql
@@ -0,0 +1,26 @@
+/*
+ * This test must be run in a database with UTF-8 encoding,
+ * because other encodings don't support all the characters used.
+ */
+
+SELECT getdatabaseencoding() <> 'UTF8'
+ AS skip_test \gset
+\if :skip_test
+\quit
+\endif
+
+set client_encoding = utf8;
+
+-- CREATE EXTENSION IF NOT EXISTS fuzzystrmatch;
+
+-- Accents
+SELECT daitch_mokotoff('Müller');
+SELECT daitch_mokotoff('Schäfer');
+SELECT daitch_mokotoff('Straßburg');
+SELECT daitch_mokotoff('Éregon');
+
+-- Special characters added at https://www.jewishgen.org/InfoFiles/Soundex.html
+SELECT daitch_mokotoff('gąszczu');
+SELECT daitch_mokotoff('brzęczy');
+SELECT daitch_mokotoff('ţamas');
+SELECT daitch_mokotoff('țamas');
diff --git a/doc/src/sgml/fuzzystrmatch.sgml b/doc/src/sgml/fuzzystrmatch.sgml
index 382e54be91..08781778f8 100644
--- a/doc/src/sgml/fuzzystrmatch.sgml
+++ b/doc/src/sgml/fuzzystrmatch.sgml
@@ -241,4 +241,101 @@ test=# SELECT dmetaphone('gumbo');
</screen>
</sect2>
+ <sect2>
+ <title>Daitch-Mokotoff Soundex</title>
+
+ <para>
+ Compared to the American Soundex System implemented in the
+ <function>soundex</function> function, the major improvements of the
+ Daitch-Mokotoff Soundex System are:
+
+ <itemizedlist spacing="compact" mark="bullet">
+ <listitem>
+ <para>
+ Information is coded to the first six meaningful letters rather than
+ four.
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ The initial letter is coded rather than kept as is.
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ Where two consecutive letters have a single sound, they are coded as a
+ single number.
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ When a letter or combination of letters may have two different sounds,
+ it is double coded under the two different codes.
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ A letter or combination of letters maps into ten possible codes rather
+ than seven.
+ </para>
+ </listitem>
+ </itemizedlist>
+ </para>
+
+ <indexterm>
+ <primary>daitch_mokotoff</primary>
+ </indexterm>
+
+ <para>
+ The following function generates Daitch-Mokotoff soundex codes for matching
+ of similar-sounding input:
+ </para>
+
+<synopsis>
+daitch_mokotoff(text source) returns text
+</synopsis>
+
+ <para>
+ Since a Daitch-Mokotoff soundex code consists of only 6 digits,
+ <literal>source</literal> should be preferably a single word or name.
+ Any alternative soundex codes are separated by space, which makes the returned
+ text suited for use in Full Text Search, see <xref linkend="textsearch"/> and
+ <xref linkend="functions-textsearch"/>.
+ </para>
+
+ <para>
+ Example:
+ </para>
+
+<programlisting>
+CREATE OR REPLACE FUNCTION soundex_name(v_name text) RETURNS text AS $$
+ SELECT string_agg(daitch_mokotoff(n), ' ')
+ FROM regexp_split_to_table(v_name, '\s+') AS n
+$$ LANGUAGE sql STRICT IMMUTABLE PARALLEL SAFE;
+
+CREATE OR REPLACE FUNCTION soundex_tsvector(v_name text) RETURNS tsvector AS $$
+ SELECT to_tsvector('simple', soundex_name(v_name))
+$$ LANGUAGE sql STRICT IMMUTABLE PARALLEL SAFE;
+
+CREATE OR REPLACE FUNCTION soundex_tsquery(v_name text) RETURNS tsquery AS $$
+ SELECT to_tsquery('simple', quote_literal(soundex_name(v_name)))
+$$ LANGUAGE sql STRICT IMMUTABLE PARALLEL SAFE;
+
+-- Note that searches could be more efficient with the tsvector in a separate column
+-- (no recalculation on table row recheck).
+CREATE TABLE s (nm text);
+CREATE INDEX ix_s_txt ON s USING gin (soundex_tsvector(nm)) WITH (fastupdate = off);
+
+INSERT INTO s VALUES ('John Doe');
+INSERT INTO s VALUES ('Jane Roe');
+INSERT INTO s VALUES ('Public John Q.');
+INSERT INTO s VALUES ('George Best');
+
+SELECT * FROM s WHERE soundex_tsvector(nm) @@ soundex_tsquery('john');
+SELECT * FROM s WHERE soundex_tsvector(nm) @@ soundex_tsquery('jane doe');
+SELECT * FROM s WHERE soundex_tsvector(nm) @@ soundex_tsquery('john public');
+SELECT * FROM s WHERE soundex_tsvector(nm) @@ soundex_tsquery('besst, giorgio');
+</programlisting>
+ </sect2>
+
</sect1>
^ permalink raw reply [nested|flat] 19+ messages in thread
* Re: daitch_mokotoff module
2022-01-03 19:16 Re: daitch_mokotoff module Andres Freund <[email protected]>
2022-01-04 13:49 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
2022-01-04 20:18 ` Re: daitch_mokotoff module Thomas Munro <[email protected]>
2022-01-05 07:05 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
2022-01-05 20:08 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
2022-02-03 14:27 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
2022-12-07 18:56 ` Re: daitch_mokotoff module Andres Freund <[email protected]>
2022-12-21 09:26 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
2022-12-22 12:00 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
@ 2022-12-22 13:27 ` Dag Lem <[email protected]>
2022-12-23 11:22 ` Re: daitch_mokotoff module Andres Freund <[email protected]>
2022-12-23 12:59 ` Re: daitch_mokotoff module Alvaro Herrera <[email protected]>
0 siblings, 2 replies; 19+ messages in thread
From: Dag Lem @ 2022-12-22 13:27 UTC (permalink / raw)
To: Andres Freund <[email protected]>; +Cc: PostgreSQL Hackers <[email protected]>; Thomas Munro <[email protected]>; Julien Rouhaud <[email protected]>
Dag Lem <[email protected]> writes:
> I noticed that the Meson builds failed in Cfbot, the updated patch adds
> a missing "include_directories" line to meson.build.
>
This should hopefully fix the last Cfbot failures, by exclusion of
daitch_mokotoff.h from headerscheck and cpluspluscheck.
Best regards
Dag Lem
Attachments:
[text/x-patch] v9-daitch_mokotoff.patch (40.4K, ../../[email protected]/2-v9-daitch_mokotoff.patch)
download | inline diff:
diff --git a/contrib/fuzzystrmatch/Makefile b/contrib/fuzzystrmatch/Makefile
index 0704894f88..12baf2d884 100644
--- a/contrib/fuzzystrmatch/Makefile
+++ b/contrib/fuzzystrmatch/Makefile
@@ -3,14 +3,15 @@
MODULE_big = fuzzystrmatch
OBJS = \
$(WIN32RES) \
+ daitch_mokotoff.o \
dmetaphone.o \
fuzzystrmatch.o
EXTENSION = fuzzystrmatch
-DATA = fuzzystrmatch--1.1.sql fuzzystrmatch--1.0--1.1.sql
+DATA = fuzzystrmatch--1.1.sql fuzzystrmatch--1.1--1.2.sql fuzzystrmatch--1.0--1.1.sql
PGFILEDESC = "fuzzystrmatch - similarities and distance between strings"
-REGRESS = fuzzystrmatch
+REGRESS = fuzzystrmatch fuzzystrmatch_utf8
ifdef USE_PGXS
PG_CONFIG = pg_config
@@ -22,3 +23,14 @@ top_builddir = ../..
include $(top_builddir)/src/Makefile.global
include $(top_srcdir)/contrib/contrib-global.mk
endif
+
+# Force this dependency to be known even without dependency info built:
+daitch_mokotoff.o: daitch_mokotoff.h
+
+daitch_mokotoff.h: daitch_mokotoff_header.pl
+ perl $< $@
+
+distprep: daitch_mokotoff.h
+
+maintainer-clean:
+ rm -f daitch_mokotoff.h
diff --git a/contrib/fuzzystrmatch/daitch_mokotoff.c b/contrib/fuzzystrmatch/daitch_mokotoff.c
new file mode 100644
index 0000000000..d4ad95c283
--- /dev/null
+++ b/contrib/fuzzystrmatch/daitch_mokotoff.c
@@ -0,0 +1,596 @@
+/*
+ * Daitch-Mokotoff Soundex
+ *
+ * Copyright (c) 2021 Finance Norway
+ * Author: Dag Lem <[email protected]>
+ *
+ * This implementation of the Daitch-Mokotoff Soundex System aims at high
+ * performance.
+ *
+ * - The processing of each phoneme is initiated by an O(1) table lookup.
+ * - For phonemes containing more than one character, a coding tree is traversed
+ * to process the complete phoneme.
+ * - The (alternate) soundex codes are produced digit by digit in-place in
+ * another tree structure.
+ *
+ * References:
+ *
+ * https://www.avotaynu.com/soundex.htm
+ * https://www.jewishgen.org/InfoFiles/Soundex.html
+ * https://familypedia.fandom.com/wiki/Daitch-Mokotoff_Soundex
+ * https://stevemorse.org/census/soundex.html (dmlat.php, dmsoundex.php)
+ * https://github.com/apache/commons-codec/ (dmrules.txt, DaitchMokotoffSoundex.java)
+ * https://metacpan.org/pod/Text::Phonetic (DaitchMokotoff.pm)
+ *
+ * A few notes on other implementations:
+ *
+ * - All other known implementations have the same unofficial rules for "UE",
+ * these are also adapted by this implementation (0, 1, NC).
+ * - The only other known implementation which is capable of generating all
+ * correct soundex codes in all cases is the JOS Soundex Calculator at
+ * https://www.jewishgen.org/jos/jossound.htm
+ * - "J" is considered (only) a vowel in dmlat.php
+ * - The official rules for "RS" are commented out in dmlat.php
+ * - Identical code digits for adjacent letters are not collapsed correctly in
+ * dmsoundex.php when double digit codes are involved. E.g. "BESST" yields
+ * 744300 instead of 743000 as for "BEST".
+ * - "J" is considered (only) a consonant in DaitchMokotoffSoundex.java
+ * - "Y" is not considered a vowel in DaitchMokotoffSoundex.java
+ *
+ * Permission to use, copy, modify, and distribute this software and its
+ * documentation for any purpose, without fee, and without a written agreement
+ * is hereby granted, provided that the above copyright notice and this
+ * paragraph and the following two paragraphs appear in all copies.
+ *
+ * IN NO EVENT SHALL THE AUTHOR OR DISTRIBUTORS BE LIABLE TO ANY PARTY FOR
+ * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING
+ * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS
+ * DOCUMENTATION, EVEN IF THE AUTHOR OR DISTRIBUTORS HAVE BEEN ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ * THE AUTHOR AND DISTRIBUTORS SPECIFICALLY DISCLAIM ANY WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
+ * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS
+ * ON AN "AS IS" BASIS, AND THE AUTHOR AND DISTRIBUTORS HAS NO OBLIGATIONS TO
+ * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
+*/
+
+#include "postgres.h"
+
+#include "lib/stringinfo.h"
+#include "mb/pg_wchar.h"
+#include "utils/builtins.h"
+#include "utils/memutils.h"
+
+
+/*
+ * The soundex coding chart table is adapted from from
+ * https://www.jewishgen.org/InfoFiles/Soundex.html
+ * See daitch_mokotoff_header.pl for details.
+*/
+
+#define DM_CODE_DIGITS 6
+
+/* Coding chart table: Soundex codes */
+typedef char dm_code[2 + 1]; /* One or two sequential code digits + NUL */
+typedef dm_code dm_codes[3]; /* Start of name, before a vowel, any other */
+
+/* Coding chart table: Letter in input sequence */
+struct dm_letter
+{
+ char letter; /* Present letter in sequence */
+ const struct dm_letter *letters; /* List of possible successive letters */
+ const dm_codes *codes; /* Code sequence(s) for complete sequence */
+};
+
+typedef struct dm_letter dm_letter;
+
+/* Generated coding chart table */
+#include "daitch_mokotoff.h"
+
+/* Node in soundex code tree */
+struct dm_node
+{
+ int soundex_length; /* Length of generated soundex code */
+ char soundex[DM_CODE_DIGITS + 1]; /* Soundex code */
+ int is_leaf; /* Candidate for complete soundex code */
+ int last_update; /* Letter number for last update of node */
+ char code_digit; /* Last code digit, 0 - 9 */
+
+ /*
+ * One or two alternate code digits leading to this node. If there are two
+ * digits, one of them is always an 'X'. Repeated code digits and 'X' lead
+ * back to the same node.
+ */
+ char prev_code_digits[2];
+ /* One or two alternate code digits moving forward. */
+ char next_code_digits[2];
+ /* ORed together code index(es) used to reach current node. */
+ int prev_code_index;
+ int next_code_index;
+ /* Nodes branching out from this node. */
+ struct dm_node *children[DM_MAX_ALTERNATE_CODES + 1];
+ /* Next node in linked list. Alternating index for each iteration. */
+ struct dm_node *next[2];
+};
+
+typedef struct dm_node dm_node;
+
+
+/* Internal C implementation */
+static int daitch_mokotoff_coding(char *word, StringInfo soundex);
+
+
+PG_FUNCTION_INFO_V1(daitch_mokotoff);
+
+Datum
+daitch_mokotoff(PG_FUNCTION_ARGS)
+{
+ text *arg = PG_GETARG_TEXT_PP(0);
+ char *string;
+ StringInfoData soundex;
+ text *retval;
+ MemoryContext old_ctx,
+ tmp_ctx;
+
+ tmp_ctx = AllocSetContextCreate(CurrentMemoryContext,
+ "daitch_mokotoff temporary context",
+ ALLOCSET_DEFAULT_SIZES);
+ old_ctx = MemoryContextSwitchTo(tmp_ctx);
+
+ string = pg_server_to_any(text_to_cstring(arg), VARSIZE_ANY_EXHDR(arg), PG_UTF8);
+ initStringInfo(&soundex);
+
+ if (!daitch_mokotoff_coding(string, &soundex))
+ {
+ /* No encodable characters in input. */
+ MemoryContextSwitchTo(old_ctx);
+ MemoryContextDelete(tmp_ctx);
+ PG_RETURN_NULL();
+ }
+
+ string = pg_any_to_server(soundex.data, soundex.len, PG_UTF8);
+ MemoryContextSwitchTo(old_ctx);
+ retval = cstring_to_text(string);
+ MemoryContextDelete(tmp_ctx);
+
+ PG_RETURN_TEXT_P(retval);
+}
+
+
+/* Template for new node in soundex code tree. */
+static const dm_node start_node = {
+ .soundex_length = 0,
+ .soundex = "000000 ", /* Six digits + joining space */
+ .is_leaf = 0,
+ .last_update = 0,
+ .code_digit = '\0',
+ .prev_code_digits = {'\0', '\0'},
+ .next_code_digits = {'\0', '\0'},
+ .prev_code_index = 0,
+ .next_code_index = 0,
+ .children = {NULL},
+ .next = {NULL}
+};
+
+/* Dummy soundex codes at end of input. */
+static const dm_codes end_codes[2] =
+{
+ {
+ "X", "X", "X"
+ }
+};
+
+
+/* Initialize soundex code tree node for next code digit. */
+static void
+initialize_node(dm_node * node, int last_update)
+{
+ if (node->last_update < last_update)
+ {
+ node->prev_code_digits[0] = node->next_code_digits[0];
+ node->prev_code_digits[1] = node->next_code_digits[1];
+ node->next_code_digits[0] = '\0';
+ node->next_code_digits[1] = '\0';
+ node->prev_code_index = node->next_code_index;
+ node->next_code_index = 0;
+ node->is_leaf = 0;
+ node->last_update = last_update;
+ }
+}
+
+
+/* Update soundex code tree node with next code digit. */
+static void
+add_next_code_digit(dm_node * node, int code_index, char code_digit)
+{
+ /* OR in index 1 or 2. */
+ node->next_code_index |= code_index;
+
+ if (!node->next_code_digits[0])
+ {
+ node->next_code_digits[0] = code_digit;
+ }
+ else if (node->next_code_digits[0] != code_digit)
+ {
+ node->next_code_digits[1] = code_digit;
+ }
+}
+
+
+/* Mark soundex code tree node as leaf. */
+static void
+set_leaf(dm_node * first_node[2], dm_node * last_node[2], dm_node * node, int ix_node)
+{
+ if (!node->is_leaf)
+ {
+ node->is_leaf = 1;
+
+ if (first_node[ix_node] == NULL)
+ {
+ first_node[ix_node] = node;
+ }
+ else
+ {
+ last_node[ix_node]->next[ix_node] = node;
+ }
+
+ last_node[ix_node] = node;
+ node->next[ix_node] = NULL;
+ }
+}
+
+
+/* Find next node corresponding to code digit, or create a new node. */
+static dm_node *
+find_or_create_child_node(dm_node * parent, char code_digit, StringInfo soundex)
+{
+ dm_node **nodes;
+ dm_node *node;
+ int i;
+
+ for (nodes = parent->children, i = 0; (node = nodes[i]); i++)
+ {
+ if (node->code_digit == code_digit)
+ {
+ /* Found existing child node. Skip completed nodes. */
+ return node->soundex_length < DM_CODE_DIGITS ? node : NULL;
+ }
+ }
+
+ /* Create new child node. */
+ Assert(i < DM_MAX_ALTERNATE_CODES);
+ node = palloc(sizeof(dm_node));
+ nodes[i] = node;
+
+ *node = start_node;
+ memcpy(node->soundex, parent->soundex, sizeof(parent->soundex));
+ node->soundex_length = parent->soundex_length;
+ node->soundex[node->soundex_length++] = code_digit;
+ node->code_digit = code_digit;
+ node->next_code_index = node->prev_code_index;
+
+ if (node->soundex_length < DM_CODE_DIGITS)
+ {
+ return node;
+ }
+ else
+ {
+ /* Append completed soundex code to soundex string. */
+ appendBinaryStringInfoNT(soundex, node->soundex, DM_CODE_DIGITS + 1);
+ return NULL;
+ }
+}
+
+
+/* Update node for next code digit(s). */
+static void
+update_node(dm_node * first_node[2], dm_node * last_node[2], dm_node * node, int ix_node,
+ int letter_no, int prev_code_index, int next_code_index,
+ const char *next_code_digits, int digit_no,
+ StringInfo soundex)
+{
+ int i;
+ char next_code_digit = next_code_digits[digit_no];
+ int num_dirty_nodes = 0;
+ dm_node *dirty_nodes[2];
+
+ initialize_node(node, letter_no);
+
+ if (node->prev_code_index && !(node->prev_code_index & prev_code_index))
+ {
+ /*
+ * If the sound (vowel / consonant) of this letter encoding doesn't
+ * correspond to the coding index of the previous letter, we skip this
+ * letter encoding. Note that currently, only "J" can be either a
+ * vowel or a consonant.
+ */
+ return;
+ }
+
+ if (next_code_digit == 'X' ||
+ (digit_no == 0 &&
+ (node->prev_code_digits[0] == next_code_digit ||
+ node->prev_code_digits[1] == next_code_digit)))
+ {
+ /* The code digit is the same as one of the previous (i.e. not added). */
+ dirty_nodes[num_dirty_nodes++] = node;
+ }
+
+ if (next_code_digit != 'X' &&
+ (digit_no > 0 ||
+ node->prev_code_digits[0] != next_code_digit ||
+ node->prev_code_digits[1]))
+ {
+ /* The code digit is different from one of the previous (i.e. added). */
+ node = find_or_create_child_node(node, next_code_digit, soundex);
+ if (node)
+ {
+ initialize_node(node, letter_no);
+ dirty_nodes[num_dirty_nodes++] = node;
+ }
+ }
+
+ for (i = 0; i < num_dirty_nodes; i++)
+ {
+ /* Add code digit leading to the current node. */
+ add_next_code_digit(dirty_nodes[i], next_code_index, next_code_digit);
+
+ if (next_code_digits[++digit_no])
+ {
+ update_node(first_node, last_node, dirty_nodes[i], ix_node,
+ letter_no, prev_code_index, next_code_index,
+ next_code_digits, digit_no,
+ soundex);
+ }
+ else
+ {
+ /* Add incomplete leaf node to linked list. */
+ set_leaf(first_node, last_node, dirty_nodes[i], ix_node);
+ }
+ }
+}
+
+
+/* Update soundex tree leaf nodes. */
+static void
+update_leaves(dm_node * first_node[2], int *ix_node, int letter_no,
+ const dm_codes * codes, const dm_codes * next_codes,
+ StringInfo soundex)
+{
+ int i,
+ j,
+ code_index;
+ dm_node *node,
+ *last_node[2];
+ const dm_code *code,
+ *next_code;
+ int ix_node_next = (*ix_node + 1) & 1; /* Alternating index: 0, 1 */
+
+ /* Initialize for new linked list of leaves. */
+ first_node[ix_node_next] = NULL;
+ last_node[ix_node_next] = NULL;
+
+ /* Process all nodes. */
+ for (node = first_node[*ix_node]; node; node = node->next[*ix_node])
+ {
+ /* One or two alternate code sequences. */
+ for (i = 0; i < 2 && (code = codes[i]) && code[0][0]; i++)
+ {
+ /* Coding for previous letter - before vowel: 1, all other: 2 */
+ int prev_code_index = (code[0][0] > '1') + 1;
+
+ /* One or two alternate next code sequences. */
+ for (j = 0; j < 2 && (next_code = next_codes[j]) && next_code[0][0]; j++)
+ {
+ /* Determine which code to use. */
+ if (letter_no == 0)
+ {
+ /* This is the first letter. */
+ code_index = 0;
+ }
+ else if (next_code[0][0] <= '1')
+ {
+ /* The next letter is a vowel. */
+ code_index = 1;
+ }
+ else
+ {
+ /* All other cases. */
+ code_index = 2;
+ }
+
+ /* One or two sequential code digits. */
+ update_node(first_node, last_node, node, ix_node_next,
+ letter_no, prev_code_index, code_index,
+ code[code_index], 0,
+ soundex);
+ }
+ }
+ }
+
+ *ix_node = ix_node_next;
+}
+
+
+/* Mapping from ISO8859-1 to upper-case ASCII */
+static const char iso8859_1_to_ascii_upper[] =
+/*
+"`abcdefghijklmnopqrstuvwxyz{|}~ ¡¢£¤¥¦§¨©ª«¬ ®¯°±²³´µ¶·¸¹º»¼½¾¿ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖ×ØÙÚÛÜÝÞßàáâãäåæçèéêëìíîïðñòóôõö÷øùúûüýþÿ"
+*/
+"`ABCDEFGHIJKLMNOPQRSTUVWXYZ{|}~ ! ?AAAAAAECEEEEIIIIDNOOOOO*OUUUUYDSAAAAAAECEEEEIIIIDNOOOOO/OUUUUYDY";
+
+
+/* Return next character, converted from UTF-8 to uppercase ASCII. */
+static char
+read_char(unsigned char *str, int *ix)
+{
+ /* Substitute character for skipped code points. */
+ const char na = '\x1a';
+ pg_wchar c;
+
+ /* Decode UTF-8 character to ISO 10646 code point. */
+ str += *ix;
+ c = utf8_to_unicode(str);
+ *ix += pg_utf_mblen(str);
+
+ if (c >= (unsigned char) '[' && c <= (unsigned char) ']')
+ {
+ /* ASCII characters [, \, and ] are reserved for Ą, Ę, and Ţ/Ț. */
+ return na;
+ }
+ else if (c < 0x60)
+ {
+ /* Non-lowercase ASCII character. */
+ return c;
+ }
+ else if (c < 0x100)
+ {
+ /* ISO-8859-1 code point, converted to upper-case ASCII. */
+ return iso8859_1_to_ascii_upper[c - 0x60];
+ }
+ else
+ {
+ /* Conversion of non-ASCII characters in the coding chart. */
+ switch (c)
+ {
+ case 0x0104:
+ case 0x0105:
+ /* Ą/ą */
+ return '[';
+ case 0x0118:
+ case 0x0119:
+ /* Ę/ę */
+ return '\\';
+ case 0x0162:
+ case 0x0163:
+ case 0x021A:
+ case 0x021B:
+ /* Ţ/ţ or Ț/ț */
+ return ']';
+ default:
+ return na;
+ }
+ }
+}
+
+
+/* Read next ASCII character, skipping any characters not in [A-\]]. */
+static char
+read_valid_char(char *str, int *ix)
+{
+ char c;
+
+ while ((c = read_char((unsigned char *) str, ix)))
+ {
+ if (c >= 'A' && c <= ']')
+ {
+ break;
+ }
+ }
+
+ return c;
+}
+
+
+/* Return sound coding for "letter" (letter sequence) */
+static const dm_codes *
+read_letter(char *str, int *ix)
+{
+ char c,
+ cmp;
+ int i,
+ j;
+ const dm_letter *letters;
+ const dm_codes *codes;
+
+ /* First letter in sequence. */
+ if (!(c = read_valid_char(str, ix)))
+ {
+ return NULL;
+ }
+ letters = &letter_[c - 'A'];
+ codes = letters->codes;
+ i = *ix;
+
+ /* Any subsequent letters in sequence. */
+ while ((letters = letters->letters) && (c = read_valid_char(str, &i)))
+ {
+ for (j = 0; (cmp = letters[j].letter); j++)
+ {
+ if (cmp == c)
+ {
+ /* Letter found. */
+ letters = &letters[j];
+ if (letters->codes)
+ {
+ /* Coding for letter sequence found. */
+ codes = letters->codes;
+ *ix = i;
+ }
+ break;
+ }
+ }
+ if (!cmp)
+ {
+ /* The sequence of letters has no coding. */
+ break;
+ }
+ }
+
+ return codes;
+}
+
+
+/* Generate all Daitch-Mokotoff soundex codes for word, separated by space. */
+static int
+daitch_mokotoff_coding(char *word, StringInfo soundex)
+{
+ int i = 0;
+ int letter_no = 0;
+ int ix_node = 0;
+ const dm_codes *codes,
+ *next_codes;
+ dm_node *first_node[2],
+ *node;
+
+ /* First letter. */
+ if (!(codes = read_letter(word, &i)))
+ {
+ /* No encodable character in input. */
+ return 0;
+ }
+
+ /* Starting point. */
+ first_node[ix_node] = palloc(sizeof(dm_node));
+ *first_node[ix_node] = start_node;
+
+ /*
+ * Loop until either the word input is exhausted, or all generated soundex
+ * codes are completed to six digits.
+ */
+ while (codes && first_node[ix_node])
+ {
+ next_codes = read_letter(word, &i);
+
+ /* Update leaf nodes. */
+ update_leaves(first_node, &ix_node, letter_no,
+ codes, next_codes ? next_codes : end_codes,
+ soundex);
+
+ codes = next_codes;
+ letter_no++;
+ }
+
+ /* Append all remaining (incomplete) soundex codes. */
+ for (node = first_node[ix_node]; node; node = node->next[ix_node])
+ {
+ appendBinaryStringInfoNT(soundex, node->soundex, DM_CODE_DIGITS + 1);
+ }
+
+ /* Terminate string at the final space. */
+ soundex->len--;
+ soundex->data[soundex->len] = '\0';
+
+ return 1;
+}
diff --git a/contrib/fuzzystrmatch/daitch_mokotoff_header.pl b/contrib/fuzzystrmatch/daitch_mokotoff_header.pl
new file mode 100755
index 0000000000..807b5fb8c5
--- /dev/null
+++ b/contrib/fuzzystrmatch/daitch_mokotoff_header.pl
@@ -0,0 +1,260 @@
+#!/bin/perl
+#
+# Generation of types and lookup tables for Daitch-Mokotoff soundex.
+#
+# Copyright (c) 2021 Finance Norway
+# Author: Dag Lem <[email protected]>
+#
+# Permission to use, copy, modify, and distribute this software and its
+# documentation for any purpose, without fee, and without a written agreement
+# is hereby granted, provided that the above copyright notice and this
+# paragraph and the following two paragraphs appear in all copies.
+#
+# IN NO EVENT SHALL THE AUTHOR OR DISTRIBUTORS BE LIABLE TO ANY PARTY FOR
+# DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING
+# LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS
+# DOCUMENTATION, EVEN IF THE AUTHOR OR DISTRIBUTORS HAVE BEEN ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+#
+# THE AUTHOR AND DISTRIBUTORS SPECIFICALLY DISCLAIM ANY WARRANTIES,
+# INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
+# AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS
+# ON AN "AS IS" BASIS, AND THE AUTHOR AND DISTRIBUTORS HAS NO OBLIGATIONS TO
+# PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
+#
+
+use strict;
+use warnings;
+use utf8;
+use open IO => ':utf8', ':std';
+use Data::Dumper;
+
+die "Usage: $0 OUTPUT_FILE\n" if @ARGV != 1;
+my $output_file = $ARGV[0];
+
+# Open the output file
+open my $OUTPUT, '>', $output_file
+ or die "Could not open output file $output_file: $!\n";
+
+# Parse code table and generate tree for letter transitions.
+my %codes;
+my %alternates;
+my $table = [{}, [["","",""]]];
+while (<DATA>) {
+ chomp;
+ my ($letters, $codes) = split(/\s+/);
+ my @codes = map { [ split(/,/) ] } split(/\|/, $codes);
+
+ # Find alternate code transitions for calculation of storage.
+ # The first character ("start of a name") can never yield more than two alternate codes,
+ # and is not considered here.
+ if (@codes > 1) {
+ for my $j (1..2) { # Codes for "before a vowel" and "any other"
+ for my $i (0..1) { # Alternate codes
+ # Identical code digits for adjacent letters are collapsed.
+ # For each possible non-transition due to code digit
+ # collapsing, find all alternate transitions.
+ my ($present, $next) = ($codes[$i][$j], $codes[($i + 1)%2][$j]);
+ next if length($present) != 1;
+ $next = $present ne substr($next, 0, 1) ? substr($next, 0, 1) : substr($next, -1, 1);
+ $alternates{$present}{$next} = 1;
+ }
+ }
+ }
+ my $key = "codes_" . join("_or_", map { join("_", @$_) } @codes);
+ my $val = join(",\n", map { "\t{\n\t\t" . join(", ", map { "\"$_\"" } @$_) . "\n\t}" } @codes);
+ $codes{$key} = $val;
+
+ for my $letter (split(/,/, $letters)) {
+ my $ref = $table->[0];
+ # Link each character to the next in the letter combination.
+ my @c = split(//, $letter);
+ my $last_c = pop(@c);
+ for my $c (@c) {
+ $ref->{$c} //= [ {}, undef ];
+ $ref->{$c}[0] //= {};
+ $ref = $ref->{$c}[0];
+ }
+ # The sound code for the letter combination is stored at the last character.
+ $ref->{$last_c}[1] = $key;
+ }
+}
+close(DATA);
+
+# Add alternates by following transitions to 'X' (not coded).
+my $alt_x = $alternates{"X"};
+delete $alt_x->{"X"};
+while (my ($k, $v) = each %alternates) {
+ if (delete $v->{"X"}) {
+ for my $x (keys %$alt_x) {
+ $v->{$x} = 1;
+ }
+ }
+}
+
+# Find the maximum number of alternate codes in one position.
+# Add two for any additional final code digit transitions.
+my $max_alt = (sort { $b <=> $a } (map { scalar keys %$_ } values %alternates))[0] + 2;
+
+print $OUTPUT <<EOF;
+/*
+ * Constants and lookup tables for Daitch-Mokotoff Soundex
+ *
+ * Copyright (c) 2021 Finance Norway
+ * Author: Dag Lem <dag\@nimrod.no>
+ *
+ * This file is generated by daitch_mokotoff_header.pl
+ *
+ * Permission to use, copy, modify, and distribute this software and its
+ * documentation for any purpose, without fee, and without a written agreement
+ * is hereby granted, provided that the above copyright notice and this
+ * paragraph and the following two paragraphs appear in all copies.
+ *
+ * IN NO EVENT SHALL THE AUTHOR OR DISTRIBUTORS BE LIABLE TO ANY PARTY FOR
+ * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING
+ * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS
+ * DOCUMENTATION, EVEN IF THE AUTHOR OR DISTRIBUTORS HAVE BEEN ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ * THE AUTHOR AND DISTRIBUTORS SPECIFICALLY DISCLAIM ANY WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
+ * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS
+ * ON AN "AS IS" BASIS, AND THE AUTHOR AND DISTRIBUTORS HAS NO OBLIGATIONS TO
+ * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
+ */
+
+#define DM_MAX_ALTERNATE_CODES $max_alt
+
+/* Codes for letter sequence at start of name, before a vowel, and any other. */
+EOF
+
+for my $key (sort keys %codes) {
+ print $OUTPUT "static const dm_codes $key\[2\] =\n{\n" . $codes{$key} . "\n};\n";
+}
+
+print $OUTPUT <<EOF;
+
+/* Coding for alternative following letters in sequence. */
+EOF
+
+sub hash2code {
+ my ($ref, $letter) = @_;
+
+ my @letters = ();
+
+ my $h = $ref->[0];
+ for my $key (sort keys %$h) {
+ $ref = $h->{$key};
+ my $children = "NULL";
+ if (defined $ref->[0]) {
+ $children = "letter_$letter$key";
+ hash2code($ref, "$letter$key");
+ }
+ my $codes = $ref->[1] // "NULL";
+ push(@letters, "\t{\n\t\t'$key', $children, $codes\n\t}");
+ }
+
+ print $OUTPUT "static const dm_letter letter_$letter\[\] =\n{\n";
+ for (@letters) {
+ print $OUTPUT "$_,\n";
+ }
+ print $OUTPUT "\t{\n\t\t'\\0'\n\t}\n";
+ print $OUTPUT "};\n";
+}
+
+hash2code($table, '');
+
+close $OUTPUT;
+
+# Table adapted from https://www.jewishgen.org/InfoFiles/Soundex.html
+#
+# The conversion from the coding chart to the table should be self
+# explanatory, but note the differences stated below.
+#
+# X = NC (not coded)
+#
+# The non-ASCII letters in the coding chart are coded with substitute
+# lowercase ASCII letters, which sort after the uppercase ASCII letters:
+#
+# Ą => a (use '[' for table lookup)
+# Ę => e (use '\\' for table lookup)
+# Ţ => t (use ']' for table lookup)
+#
+# The rule for "UE" does not correspond to the coding chart, however
+# it is used by all other known implementations, including the one at
+# https://www.jewishgen.org/jos/jossound.htm (try e.g. "bouey").
+#
+# Note that the implementation assumes that vowels are assigned code
+# 0 or 1. "J" can be either a vowel or a consonant.
+#
+
+__DATA__
+AI,AJ,AY 0,1,X
+AU 0,7,X
+a X,X,6|X,X,X
+A 0,X,X
+B 7,7,7
+CHS 5,54,54
+CH 5,5,5|4,4,4
+CK 5,5,5|45,45,45
+CZ,CS,CSZ,CZS 4,4,4
+C 5,5,5|4,4,4
+DRZ,DRS 4,4,4
+DS,DSH,DSZ 4,4,4
+DZ,DZH,DZS 4,4,4
+D,DT 3,3,3
+EI,EJ,EY 0,1,X
+EU 1,1,X
+e X,X,6|X,X,X
+E 0,X,X
+FB 7,7,7
+F 7,7,7
+G 5,5,5
+H 5,5,X
+IA,IE,IO,IU 1,X,X
+I 0,X,X
+J 1,X,X|4,4,4
+KS 5,54,54
+KH 5,5,5
+K 5,5,5
+L 8,8,8
+MN 66,66,66
+M 6,6,6
+NM 66,66,66
+N 6,6,6
+OI,OJ,OY 0,1,X
+O 0,X,X
+P,PF,PH 7,7,7
+Q 5,5,5
+RZ,RS 94,94,94|4,4,4
+R 9,9,9
+SCHTSCH,SCHTSH,SCHTCH 2,4,4
+SCH 4,4,4
+SHTCH,SHCH,SHTSH 2,4,4
+SHT,SCHT,SCHD 2,43,43
+SH 4,4,4
+STCH,STSCH,SC 2,4,4
+STRZ,STRS,STSH 2,4,4
+ST 2,43,43
+SZCZ,SZCS 2,4,4
+SZT,SHD,SZD,SD 2,43,43
+SZ 4,4,4
+S 4,4,4
+TCH,TTCH,TTSCH 4,4,4
+TH 3,3,3
+TRZ,TRS 4,4,4
+TSCH,TSH 4,4,4
+TS,TTS,TTSZ,TC 4,4,4
+TZ,TTZ,TZS,TSZ 4,4,4
+t 3,3,3|4,4,4
+T 3,3,3
+UI,UJ,UY,UE 0,1,X
+U 0,X,X
+V 7,7,7
+W 7,7,7
+X 5,54,54
+Y 1,X,X
+ZDZ,ZDZH,ZHDZH 2,4,4
+ZD,ZHD 2,43,43
+ZH,ZS,ZSCH,ZSH 4,4,4
+Z 4,4,4
diff --git a/contrib/fuzzystrmatch/expected/fuzzystrmatch.out b/contrib/fuzzystrmatch/expected/fuzzystrmatch.out
index 493c95cdfa..f62ddad4ee 100644
--- a/contrib/fuzzystrmatch/expected/fuzzystrmatch.out
+++ b/contrib/fuzzystrmatch/expected/fuzzystrmatch.out
@@ -65,3 +65,174 @@ SELECT dmetaphone_alt('gumbo');
KMP
(1 row)
+-- Wovels
+SELECT daitch_mokotoff('Augsburg');
+ daitch_mokotoff
+-----------------
+ 054795
+(1 row)
+
+SELECT daitch_mokotoff('Breuer');
+ daitch_mokotoff
+-----------------
+ 791900
+(1 row)
+
+SELECT daitch_mokotoff('Freud');
+ daitch_mokotoff
+-----------------
+ 793000
+(1 row)
+
+-- The letter "H"
+SELECT daitch_mokotoff('Halberstadt');
+ daitch_mokotoff
+-----------------
+ 587943 587433
+(1 row)
+
+SELECT daitch_mokotoff('Mannheim');
+ daitch_mokotoff
+-----------------
+ 665600
+(1 row)
+
+-- Adjacent sounds
+SELECT daitch_mokotoff('Chernowitz');
+ daitch_mokotoff
+-----------------
+ 596740 496740
+(1 row)
+
+-- Adjacent letters with identical adjacent code digits
+SELECT daitch_mokotoff('Cherkassy');
+ daitch_mokotoff
+-----------------
+ 595400 495400
+(1 row)
+
+SELECT daitch_mokotoff('Kleinman');
+ daitch_mokotoff
+-----------------
+ 586660
+(1 row)
+
+-- More than one word
+SELECT daitch_mokotoff('Nowy Targ');
+ daitch_mokotoff
+-----------------
+ 673950
+(1 row)
+
+-- Padded with "0"
+SELECT daitch_mokotoff('Berlin');
+ daitch_mokotoff
+-----------------
+ 798600
+(1 row)
+
+-- Other examples from https://www.avotaynu.com/soundex.htm
+SELECT daitch_mokotoff('Ceniow');
+ daitch_mokotoff
+-----------------
+ 567000 467000
+(1 row)
+
+SELECT daitch_mokotoff('Tsenyuv');
+ daitch_mokotoff
+-----------------
+ 467000
+(1 row)
+
+SELECT daitch_mokotoff('Holubica');
+ daitch_mokotoff
+-----------------
+ 587500 587400
+(1 row)
+
+SELECT daitch_mokotoff('Golubitsa');
+ daitch_mokotoff
+-----------------
+ 587400
+(1 row)
+
+SELECT daitch_mokotoff('Przemysl');
+ daitch_mokotoff
+-----------------
+ 794648 746480
+(1 row)
+
+SELECT daitch_mokotoff('Pshemeshil');
+ daitch_mokotoff
+-----------------
+ 746480
+(1 row)
+
+SELECT daitch_mokotoff('Rosochowaciec');
+ daitch_mokotoff
+---------------------------------------------------------
+ 945755 945754 945745 945744 944755 944754 944745 944744
+(1 row)
+
+SELECT daitch_mokotoff('Rosokhovatsets');
+ daitch_mokotoff
+-----------------
+ 945744
+(1 row)
+
+-- Ignored characters
+SELECT daitch_mokotoff('''OBrien');
+ daitch_mokotoff
+-----------------
+ 079600
+(1 row)
+
+SELECT daitch_mokotoff('O''Brien');
+ daitch_mokotoff
+-----------------
+ 079600
+(1 row)
+
+-- "Difficult" cases, likely to cause trouble for other implementations.
+SELECT daitch_mokotoff('CJC');
+ daitch_mokotoff
+-------------------------------------------
+ 550000 540000 545000 450000 400000 440000
+(1 row)
+
+SELECT daitch_mokotoff('BESST');
+ daitch_mokotoff
+-----------------
+ 743000
+(1 row)
+
+SELECT daitch_mokotoff('BOUEY');
+ daitch_mokotoff
+-----------------
+ 710000
+(1 row)
+
+SELECT daitch_mokotoff('HANNMANN');
+ daitch_mokotoff
+-----------------
+ 566600
+(1 row)
+
+SELECT daitch_mokotoff('MCCOYJR');
+ daitch_mokotoff
+---------------------------------------------------------
+ 651900 654900 654190 654490 645190 645490 641900 644900
+(1 row)
+
+SELECT daitch_mokotoff('ACCURSO');
+ daitch_mokotoff
+---------------------------------------------------------
+ 059400 054000 054940 054400 045940 045400 049400 044000
+(1 row)
+
+SELECT daitch_mokotoff('BIERSCHBACH');
+ daitch_mokotoff
+---------------------------------------------------------
+ 794575 794574 794750 794740 745750 745740 747500 747400
+(1 row)
+
diff --git a/contrib/fuzzystrmatch/expected/fuzzystrmatch_utf8.out b/contrib/fuzzystrmatch/expected/fuzzystrmatch_utf8.out
new file mode 100644
index 0000000000..32d8260383
--- /dev/null
+++ b/contrib/fuzzystrmatch/expected/fuzzystrmatch_utf8.out
@@ -0,0 +1,61 @@
+/*
+ * This test must be run in a database with UTF-8 encoding,
+ * because other encodings don't support all the characters used.
+ */
+SELECT getdatabaseencoding() <> 'UTF8'
+ AS skip_test \gset
+\if :skip_test
+\quit
+\endif
+set client_encoding = utf8;
+-- CREATE EXTENSION IF NOT EXISTS fuzzystrmatch;
+-- Accents
+SELECT daitch_mokotoff('Müller');
+ daitch_mokotoff
+-----------------
+ 689000
+(1 row)
+
+SELECT daitch_mokotoff('Schäfer');
+ daitch_mokotoff
+-----------------
+ 479000
+(1 row)
+
+SELECT daitch_mokotoff('Straßburg');
+ daitch_mokotoff
+-----------------
+ 294795
+(1 row)
+
+SELECT daitch_mokotoff('Éregon');
+ daitch_mokotoff
+-----------------
+ 095600
+(1 row)
+
+-- Special characters added at https://www.jewishgen.org/InfoFiles/Soundex.html
+SELECT daitch_mokotoff('gąszczu');
+ daitch_mokotoff
+-----------------
+ 564000 540000
+(1 row)
+
+SELECT daitch_mokotoff('brzęczy');
+ daitch_mokotoff
+-----------------------------
+ 794640 794400 746400 744000
+(1 row)
+
+SELECT daitch_mokotoff('ţamas');
+ daitch_mokotoff
+-----------------
+ 364000 464000
+(1 row)
+
+SELECT daitch_mokotoff('țamas');
+ daitch_mokotoff
+-----------------
+ 364000 464000
+(1 row)
+
diff --git a/contrib/fuzzystrmatch/expected/fuzzystrmatch_utf8_1.out b/contrib/fuzzystrmatch/expected/fuzzystrmatch_utf8_1.out
new file mode 100644
index 0000000000..37aead89c0
--- /dev/null
+++ b/contrib/fuzzystrmatch/expected/fuzzystrmatch_utf8_1.out
@@ -0,0 +1,8 @@
+/*
+ * This test must be run in a database with UTF-8 encoding,
+ * because other encodings don't support all the characters used.
+ */
+SELECT getdatabaseencoding() <> 'UTF8'
+ AS skip_test \gset
+\if :skip_test
+\quit
diff --git a/contrib/fuzzystrmatch/fuzzystrmatch--1.1--1.2.sql b/contrib/fuzzystrmatch/fuzzystrmatch--1.1--1.2.sql
new file mode 100644
index 0000000000..b9d7b229a3
--- /dev/null
+++ b/contrib/fuzzystrmatch/fuzzystrmatch--1.1--1.2.sql
@@ -0,0 +1,8 @@
+/* contrib/fuzzystrmatch/fuzzystrmatch--1.1--1.2.sql */
+
+-- complain if script is sourced in psql, rather than via ALTER EXTENSION
+\echo Use "ALTER EXTENSION fuzzystrmatch UPDATE TO '1.2'" to load this file. \quit
+
+CREATE FUNCTION daitch_mokotoff(text) RETURNS text
+AS 'MODULE_PATHNAME', 'daitch_mokotoff'
+LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
diff --git a/contrib/fuzzystrmatch/fuzzystrmatch.control b/contrib/fuzzystrmatch/fuzzystrmatch.control
index 3cd6660bf9..8b6e9fd993 100644
--- a/contrib/fuzzystrmatch/fuzzystrmatch.control
+++ b/contrib/fuzzystrmatch/fuzzystrmatch.control
@@ -1,6 +1,6 @@
# fuzzystrmatch extension
comment = 'determine similarities and distance between strings'
-default_version = '1.1'
+default_version = '1.2'
module_pathname = '$libdir/fuzzystrmatch'
relocatable = true
trusted = true
diff --git a/contrib/fuzzystrmatch/meson.build b/contrib/fuzzystrmatch/meson.build
index e6d06149ce..73178794c2 100644
--- a/contrib/fuzzystrmatch/meson.build
+++ b/contrib/fuzzystrmatch/meson.build
@@ -1,7 +1,16 @@
fuzzystrmatch_sources = files(
- 'fuzzystrmatch.c',
+ 'daitch_mokotoff.c',
'dmetaphone.c',
+ 'fuzzystrmatch.c',
+)
+
+daitch_mokotoff_h = custom_target('daitch_mokotoff',
+ input: 'daitch_mokotoff_header.pl',
+ output: 'daitch_mokotoff.h',
+ command: [perl, '@INPUT@', '@OUTPUT@'],
)
+generated_sources += daitch_mokotoff_h
+fuzzystrmatch_sources += daitch_mokotoff_h
if host_system == 'windows'
fuzzystrmatch_sources += rc_lib_gen.process(win32ver_rc, extra_args: [
@@ -11,6 +20,7 @@ endif
fuzzystrmatch = shared_module('fuzzystrmatch',
fuzzystrmatch_sources,
+ include_directories: include_directories('.'),
kwargs: contrib_mod_args,
)
contrib_targets += fuzzystrmatch
@@ -19,6 +29,7 @@ install_data(
'fuzzystrmatch.control',
'fuzzystrmatch--1.0--1.1.sql',
'fuzzystrmatch--1.1.sql',
+ 'fuzzystrmatch--1.1--1.2.sql',
kwargs: contrib_data_args,
)
@@ -29,6 +40,7 @@ tests += {
'regress': {
'sql': [
'fuzzystrmatch',
+ 'fuzzystrmatch_utf8',
],
},
}
diff --git a/contrib/fuzzystrmatch/sql/fuzzystrmatch.sql b/contrib/fuzzystrmatch/sql/fuzzystrmatch.sql
index f05dc28ffb..db05c7d6b6 100644
--- a/contrib/fuzzystrmatch/sql/fuzzystrmatch.sql
+++ b/contrib/fuzzystrmatch/sql/fuzzystrmatch.sql
@@ -19,3 +19,48 @@ SELECT metaphone('GUMBO', 4);
SELECT dmetaphone('gumbo');
SELECT dmetaphone_alt('gumbo');
+
+-- Wovels
+SELECT daitch_mokotoff('Augsburg');
+SELECT daitch_mokotoff('Breuer');
+SELECT daitch_mokotoff('Freud');
+
+-- The letter "H"
+SELECT daitch_mokotoff('Halberstadt');
+SELECT daitch_mokotoff('Mannheim');
+
+-- Adjacent sounds
+SELECT daitch_mokotoff('Chernowitz');
+
+-- Adjacent letters with identical adjacent code digits
+SELECT daitch_mokotoff('Cherkassy');
+SELECT daitch_mokotoff('Kleinman');
+
+-- More than one word
+SELECT daitch_mokotoff('Nowy Targ');
+
+-- Padded with "0"
+SELECT daitch_mokotoff('Berlin');
+
+-- Other examples from https://www.avotaynu.com/soundex.htm
+SELECT daitch_mokotoff('Ceniow');
+SELECT daitch_mokotoff('Tsenyuv');
+SELECT daitch_mokotoff('Holubica');
+SELECT daitch_mokotoff('Golubitsa');
+SELECT daitch_mokotoff('Przemysl');
+SELECT daitch_mokotoff('Pshemeshil');
+SELECT daitch_mokotoff('Rosochowaciec');
+SELECT daitch_mokotoff('Rosokhovatsets');
+
+-- Ignored characters
+SELECT daitch_mokotoff('''OBrien');
+SELECT daitch_mokotoff('O''Brien');
+
+-- "Difficult" cases, likely to cause trouble for other implementations.
+SELECT daitch_mokotoff('CJC');
+SELECT daitch_mokotoff('BESST');
+SELECT daitch_mokotoff('BOUEY');
+SELECT daitch_mokotoff('HANNMANN');
+SELECT daitch_mokotoff('MCCOYJR');
+SELECT daitch_mokotoff('ACCURSO');
+SELECT daitch_mokotoff('BIERSCHBACH');
diff --git a/contrib/fuzzystrmatch/sql/fuzzystrmatch_utf8.sql b/contrib/fuzzystrmatch/sql/fuzzystrmatch_utf8.sql
new file mode 100644
index 0000000000..f42c01a1bb
--- /dev/null
+++ b/contrib/fuzzystrmatch/sql/fuzzystrmatch_utf8.sql
@@ -0,0 +1,26 @@
+/*
+ * This test must be run in a database with UTF-8 encoding,
+ * because other encodings don't support all the characters used.
+ */
+
+SELECT getdatabaseencoding() <> 'UTF8'
+ AS skip_test \gset
+\if :skip_test
+\quit
+\endif
+
+set client_encoding = utf8;
+
+-- CREATE EXTENSION IF NOT EXISTS fuzzystrmatch;
+
+-- Accents
+SELECT daitch_mokotoff('Müller');
+SELECT daitch_mokotoff('Schäfer');
+SELECT daitch_mokotoff('Straßburg');
+SELECT daitch_mokotoff('Éregon');
+
+-- Special characters added at https://www.jewishgen.org/InfoFiles/Soundex.html
+SELECT daitch_mokotoff('gąszczu');
+SELECT daitch_mokotoff('brzęczy');
+SELECT daitch_mokotoff('ţamas');
+SELECT daitch_mokotoff('țamas');
diff --git a/doc/src/sgml/fuzzystrmatch.sgml b/doc/src/sgml/fuzzystrmatch.sgml
index 382e54be91..08781778f8 100644
--- a/doc/src/sgml/fuzzystrmatch.sgml
+++ b/doc/src/sgml/fuzzystrmatch.sgml
@@ -241,4 +241,101 @@ test=# SELECT dmetaphone('gumbo');
</screen>
</sect2>
+ <sect2>
+ <title>Daitch-Mokotoff Soundex</title>
+
+ <para>
+ Compared to the American Soundex System implemented in the
+ <function>soundex</function> function, the major improvements of the
+ Daitch-Mokotoff Soundex System are:
+
+ <itemizedlist spacing="compact" mark="bullet">
+ <listitem>
+ <para>
+ Information is coded to the first six meaningful letters rather than
+ four.
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ The initial letter is coded rather than kept as is.
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ Where two consecutive letters have a single sound, they are coded as a
+ single number.
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ When a letter or combination of letters may have two different sounds,
+ it is double coded under the two different codes.
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ A letter or combination of letters maps into ten possible codes rather
+ than seven.
+ </para>
+ </listitem>
+ </itemizedlist>
+ </para>
+
+ <indexterm>
+ <primary>daitch_mokotoff</primary>
+ </indexterm>
+
+ <para>
+ The following function generates Daitch-Mokotoff soundex codes for matching
+ of similar-sounding input:
+ </para>
+
+<synopsis>
+daitch_mokotoff(text source) returns text
+</synopsis>
+
+ <para>
+ Since a Daitch-Mokotoff soundex code consists of only 6 digits,
+ <literal>source</literal> should be preferably a single word or name.
+ Any alternative soundex codes are separated by space, which makes the returned
+ text suited for use in Full Text Search, see <xref linkend="textsearch"/> and
+ <xref linkend="functions-textsearch"/>.
+ </para>
+
+ <para>
+ Example:
+ </para>
+
+<programlisting>
+CREATE OR REPLACE FUNCTION soundex_name(v_name text) RETURNS text AS $$
+ SELECT string_agg(daitch_mokotoff(n), ' ')
+ FROM regexp_split_to_table(v_name, '\s+') AS n
+$$ LANGUAGE sql STRICT IMMUTABLE PARALLEL SAFE;
+
+CREATE OR REPLACE FUNCTION soundex_tsvector(v_name text) RETURNS tsvector AS $$
+ SELECT to_tsvector('simple', soundex_name(v_name))
+$$ LANGUAGE sql STRICT IMMUTABLE PARALLEL SAFE;
+
+CREATE OR REPLACE FUNCTION soundex_tsquery(v_name text) RETURNS tsquery AS $$
+ SELECT to_tsquery('simple', quote_literal(soundex_name(v_name)))
+$$ LANGUAGE sql STRICT IMMUTABLE PARALLEL SAFE;
+
+-- Note that searches could be more efficient with the tsvector in a separate column
+-- (no recalculation on table row recheck).
+CREATE TABLE s (nm text);
+CREATE INDEX ix_s_txt ON s USING gin (soundex_tsvector(nm)) WITH (fastupdate = off);
+
+INSERT INTO s VALUES ('John Doe');
+INSERT INTO s VALUES ('Jane Roe');
+INSERT INTO s VALUES ('Public John Q.');
+INSERT INTO s VALUES ('George Best');
+
+SELECT * FROM s WHERE soundex_tsvector(nm) @@ soundex_tsquery('john');
+SELECT * FROM s WHERE soundex_tsvector(nm) @@ soundex_tsquery('jane doe');
+SELECT * FROM s WHERE soundex_tsvector(nm) @@ soundex_tsquery('john public');
+SELECT * FROM s WHERE soundex_tsvector(nm) @@ soundex_tsquery('besst, giorgio');
+</programlisting>
+ </sect2>
+
</sect1>
diff --git a/src/tools/pginclude/cpluspluscheck b/src/tools/pginclude/cpluspluscheck
index b393f2a2ea..17a37aee85 100755
--- a/src/tools/pginclude/cpluspluscheck
+++ b/src/tools/pginclude/cpluspluscheck
@@ -119,6 +119,9 @@ do
test "$f" = src/include/common/unicode_nonspacing_table.h && continue
test "$f" = src/include/common/unicode_east_asian_fw_table.h && continue
+ # Also not meant to be included standalone.
+ test "$f" = contrib/fuzzystrmatch/daitch_mokotoff.h && continue
+
# We can't make these Bison output files compilable standalone
# without using "%code require", which old Bison versions lack.
# parser/gram.h will be included by parser/gramparse.h anyway.
diff --git a/src/tools/pginclude/headerscheck b/src/tools/pginclude/headerscheck
index 2a39856f88..24aacd1239 100755
--- a/src/tools/pginclude/headerscheck
+++ b/src/tools/pginclude/headerscheck
@@ -114,6 +114,9 @@ do
test "$f" = src/include/common/unicode_nonspacing_table.h && continue
test "$f" = src/include/common/unicode_east_asian_fw_table.h && continue
+ # Also not meant to be included standalone.
+ test "$f" = contrib/fuzzystrmatch/daitch_mokotoff.h && continue
+
# We can't make these Bison output files compilable standalone
# without using "%code require", which old Bison versions lack.
# parser/gram.h will be included by parser/gramparse.h anyway.
^ permalink raw reply [nested|flat] 19+ messages in thread
* Re: daitch_mokotoff module
2022-01-03 19:16 Re: daitch_mokotoff module Andres Freund <[email protected]>
2022-01-04 13:49 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
2022-01-04 20:18 ` Re: daitch_mokotoff module Thomas Munro <[email protected]>
2022-01-05 07:05 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
2022-01-05 20:08 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
2022-02-03 14:27 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
2022-12-07 18:56 ` Re: daitch_mokotoff module Andres Freund <[email protected]>
2022-12-21 09:26 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
2022-12-22 12:00 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
2022-12-22 13:27 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
@ 2022-12-23 11:22 ` Andres Freund <[email protected]>
2022-12-23 20:34 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
1 sibling, 1 reply; 19+ messages in thread
From: Andres Freund @ 2022-12-23 11:22 UTC (permalink / raw)
To: Dag Lem <[email protected]>; +Cc: PostgreSQL Hackers <[email protected]>; Thomas Munro <[email protected]>; Julien Rouhaud <[email protected]>
On 2022-12-22 14:27:54 +0100, Dag Lem wrote:
> This should hopefully fix the last Cfbot failures, by exclusion of
> daitch_mokotoff.h from headerscheck and cpluspluscheck.
Btw, you can do the same tests as cfbot in your own repo by enabling CI
in a github repo. See src/tools/ci/README
^ permalink raw reply [nested|flat] 19+ messages in thread
* Re: daitch_mokotoff module
2022-01-03 19:16 Re: daitch_mokotoff module Andres Freund <[email protected]>
2022-01-04 13:49 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
2022-01-04 20:18 ` Re: daitch_mokotoff module Thomas Munro <[email protected]>
2022-01-05 07:05 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
2022-01-05 20:08 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
2022-02-03 14:27 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
2022-12-07 18:56 ` Re: daitch_mokotoff module Andres Freund <[email protected]>
2022-12-21 09:26 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
2022-12-22 12:00 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
2022-12-22 13:27 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
2022-12-23 11:22 ` Re: daitch_mokotoff module Andres Freund <[email protected]>
@ 2022-12-23 20:34 ` Dag Lem <[email protected]>
0 siblings, 0 replies; 19+ messages in thread
From: Dag Lem @ 2022-12-23 20:34 UTC (permalink / raw)
To: Andres Freund <[email protected]>; +Cc: PostgreSQL Hackers <[email protected]>; Thomas Munro <[email protected]>; Julien Rouhaud <[email protected]>
Andres Freund <[email protected]> writes:
> On 2022-12-22 14:27:54 +0100, Dag Lem wrote:
>> This should hopefully fix the last Cfbot failures, by exclusion of
>> daitch_mokotoff.h from headerscheck and cpluspluscheck.
>
> Btw, you can do the same tests as cfbot in your own repo by enabling CI
> in a github repo. See src/tools/ci/README
>
OK, thanks, I've set it up now.
Best regards,
Dag Lem
^ permalink raw reply [nested|flat] 19+ messages in thread
* Re: daitch_mokotoff module
2022-01-03 19:16 Re: daitch_mokotoff module Andres Freund <[email protected]>
2022-01-04 13:49 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
2022-01-04 20:18 ` Re: daitch_mokotoff module Thomas Munro <[email protected]>
2022-01-05 07:05 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
2022-01-05 20:08 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
2022-02-03 14:27 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
2022-12-07 18:56 ` Re: daitch_mokotoff module Andres Freund <[email protected]>
2022-12-21 09:26 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
2022-12-22 12:00 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
2022-12-22 13:27 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
@ 2022-12-23 12:59 ` Alvaro Herrera <[email protected]>
2022-12-23 14:57 ` Re: daitch_mokotoff module Tom Lane <[email protected]>
1 sibling, 1 reply; 19+ messages in thread
From: Alvaro Herrera @ 2022-12-23 12:59 UTC (permalink / raw)
To: Dag Lem <[email protected]>; +Cc: Andres Freund <[email protected]>; PostgreSQL Hackers <[email protected]>; Thomas Munro <[email protected]>; Julien Rouhaud <[email protected]>
On 2022-Dec-22, Dag Lem wrote:
> This should hopefully fix the last Cfbot failures, by exclusion of
> daitch_mokotoff.h from headerscheck and cpluspluscheck.
Hmm, maybe it'd be better to move the typedefs to the .h file instead.
--
Álvaro Herrera PostgreSQL Developer — https://www.EnterpriseDB.com/
"Pensar que el espectro que vemos es ilusorio no lo despoja de espanto,
sólo le suma el nuevo terror de la locura" (Perelandra, C.S. Lewis)
^ permalink raw reply [nested|flat] 19+ messages in thread
* Re: daitch_mokotoff module
2022-01-03 19:16 Re: daitch_mokotoff module Andres Freund <[email protected]>
2022-01-04 13:49 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
2022-01-04 20:18 ` Re: daitch_mokotoff module Thomas Munro <[email protected]>
2022-01-05 07:05 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
2022-01-05 20:08 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
2022-02-03 14:27 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
2022-12-07 18:56 ` Re: daitch_mokotoff module Andres Freund <[email protected]>
2022-12-21 09:26 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
2022-12-22 12:00 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
2022-12-22 13:27 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
2022-12-23 12:59 ` Re: daitch_mokotoff module Alvaro Herrera <[email protected]>
@ 2022-12-23 14:57 ` Tom Lane <[email protected]>
2022-12-23 20:55 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
0 siblings, 1 reply; 19+ messages in thread
From: Tom Lane @ 2022-12-23 14:57 UTC (permalink / raw)
To: Alvaro Herrera <[email protected]>; +Cc: Dag Lem <[email protected]>; Andres Freund <[email protected]>; PostgreSQL Hackers <[email protected]>; Thomas Munro <[email protected]>; Julien Rouhaud <[email protected]>
Alvaro Herrera <[email protected]> writes:
> On 2022-Dec-22, Dag Lem wrote:
>> This should hopefully fix the last Cfbot failures, by exclusion of
>> daitch_mokotoff.h from headerscheck and cpluspluscheck.
> Hmm, maybe it'd be better to move the typedefs to the .h file instead.
Indeed, that sounds like exactly the wrong way to fix such a problem.
The bar for excluding stuff from headerscheck needs to be very high.
regards, tom lane
^ permalink raw reply [nested|flat] 19+ messages in thread
* Re: daitch_mokotoff module
2022-01-03 19:16 Re: daitch_mokotoff module Andres Freund <[email protected]>
2022-01-04 13:49 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
2022-01-04 20:18 ` Re: daitch_mokotoff module Thomas Munro <[email protected]>
2022-01-05 07:05 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
2022-01-05 20:08 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
2022-02-03 14:27 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
2022-12-07 18:56 ` Re: daitch_mokotoff module Andres Freund <[email protected]>
2022-12-21 09:26 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
2022-12-22 12:00 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
2022-12-22 13:27 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
2022-12-23 12:59 ` Re: daitch_mokotoff module Alvaro Herrera <[email protected]>
2022-12-23 14:57 ` Re: daitch_mokotoff module Tom Lane <[email protected]>
@ 2022-12-23 20:55 ` Dag Lem <[email protected]>
0 siblings, 0 replies; 19+ messages in thread
From: Dag Lem @ 2022-12-23 20:55 UTC (permalink / raw)
To: Tom Lane <[email protected]>; +Cc: Alvaro Herrera <[email protected]>; Andres Freund <[email protected]>; PostgreSQL Hackers <[email protected]>; Thomas Munro <[email protected]>; Julien Rouhaud <[email protected]>
Tom Lane <[email protected]> writes:
> Alvaro Herrera <[email protected]> writes:
>> On 2022-Dec-22, Dag Lem wrote:
>>> This should hopefully fix the last Cfbot failures, by exclusion of
>>> daitch_mokotoff.h from headerscheck and cpluspluscheck.
>
>> Hmm, maybe it'd be better to move the typedefs to the .h file instead.
>
> Indeed, that sounds like exactly the wrong way to fix such a problem.
> The bar for excluding stuff from headerscheck needs to be very high.
>
OK, I've moved enough declarations back to the generated header file
again so as to avoid excluding it from headerscheck and cpluspluscheck.
Best regards,
Dag Lem
Attachments:
[text/x-patch] v10-daitch_mokotoff.patch (39.1K, ../../[email protected]/2-v10-daitch_mokotoff.patch)
download | inline diff:
diff --git a/contrib/fuzzystrmatch/Makefile b/contrib/fuzzystrmatch/Makefile
index 0704894f88..12baf2d884 100644
--- a/contrib/fuzzystrmatch/Makefile
+++ b/contrib/fuzzystrmatch/Makefile
@@ -3,14 +3,15 @@
MODULE_big = fuzzystrmatch
OBJS = \
$(WIN32RES) \
+ daitch_mokotoff.o \
dmetaphone.o \
fuzzystrmatch.o
EXTENSION = fuzzystrmatch
-DATA = fuzzystrmatch--1.1.sql fuzzystrmatch--1.0--1.1.sql
+DATA = fuzzystrmatch--1.1.sql fuzzystrmatch--1.1--1.2.sql fuzzystrmatch--1.0--1.1.sql
PGFILEDESC = "fuzzystrmatch - similarities and distance between strings"
-REGRESS = fuzzystrmatch
+REGRESS = fuzzystrmatch fuzzystrmatch_utf8
ifdef USE_PGXS
PG_CONFIG = pg_config
@@ -22,3 +23,14 @@ top_builddir = ../..
include $(top_builddir)/src/Makefile.global
include $(top_srcdir)/contrib/contrib-global.mk
endif
+
+# Force this dependency to be known even without dependency info built:
+daitch_mokotoff.o: daitch_mokotoff.h
+
+daitch_mokotoff.h: daitch_mokotoff_header.pl
+ perl $< $@
+
+distprep: daitch_mokotoff.h
+
+maintainer-clean:
+ rm -f daitch_mokotoff.h
diff --git a/contrib/fuzzystrmatch/daitch_mokotoff.c b/contrib/fuzzystrmatch/daitch_mokotoff.c
new file mode 100644
index 0000000000..e809d4a39e
--- /dev/null
+++ b/contrib/fuzzystrmatch/daitch_mokotoff.c
@@ -0,0 +1,582 @@
+/*
+ * Daitch-Mokotoff Soundex
+ *
+ * Copyright (c) 2021 Finance Norway
+ * Author: Dag Lem <[email protected]>
+ *
+ * This implementation of the Daitch-Mokotoff Soundex System aims at high
+ * performance.
+ *
+ * - The processing of each phoneme is initiated by an O(1) table lookup.
+ * - For phonemes containing more than one character, a coding tree is traversed
+ * to process the complete phoneme.
+ * - The (alternate) soundex codes are produced digit by digit in-place in
+ * another tree structure.
+ *
+ * References:
+ *
+ * https://www.avotaynu.com/soundex.htm
+ * https://www.jewishgen.org/InfoFiles/Soundex.html
+ * https://familypedia.fandom.com/wiki/Daitch-Mokotoff_Soundex
+ * https://stevemorse.org/census/soundex.html (dmlat.php, dmsoundex.php)
+ * https://github.com/apache/commons-codec/ (dmrules.txt, DaitchMokotoffSoundex.java)
+ * https://metacpan.org/pod/Text::Phonetic (DaitchMokotoff.pm)
+ *
+ * A few notes on other implementations:
+ *
+ * - All other known implementations have the same unofficial rules for "UE",
+ * these are also adapted by this implementation (0, 1, NC).
+ * - The only other known implementation which is capable of generating all
+ * correct soundex codes in all cases is the JOS Soundex Calculator at
+ * https://www.jewishgen.org/jos/jossound.htm
+ * - "J" is considered (only) a vowel in dmlat.php
+ * - The official rules for "RS" are commented out in dmlat.php
+ * - Identical code digits for adjacent letters are not collapsed correctly in
+ * dmsoundex.php when double digit codes are involved. E.g. "BESST" yields
+ * 744300 instead of 743000 as for "BEST".
+ * - "J" is considered (only) a consonant in DaitchMokotoffSoundex.java
+ * - "Y" is not considered a vowel in DaitchMokotoffSoundex.java
+ *
+ * Permission to use, copy, modify, and distribute this software and its
+ * documentation for any purpose, without fee, and without a written agreement
+ * is hereby granted, provided that the above copyright notice and this
+ * paragraph and the following two paragraphs appear in all copies.
+ *
+ * IN NO EVENT SHALL THE AUTHOR OR DISTRIBUTORS BE LIABLE TO ANY PARTY FOR
+ * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING
+ * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS
+ * DOCUMENTATION, EVEN IF THE AUTHOR OR DISTRIBUTORS HAVE BEEN ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ * THE AUTHOR AND DISTRIBUTORS SPECIFICALLY DISCLAIM ANY WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
+ * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS
+ * ON AN "AS IS" BASIS, AND THE AUTHOR AND DISTRIBUTORS HAS NO OBLIGATIONS TO
+ * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
+*/
+
+#include "postgres.h"
+
+#include "lib/stringinfo.h"
+#include "mb/pg_wchar.h"
+#include "utils/builtins.h"
+#include "utils/memutils.h"
+
+
+/*
+ * The soundex coding chart table is adapted from from
+ * https://www.jewishgen.org/InfoFiles/Soundex.html
+ * See daitch_mokotoff_header.pl for details.
+*/
+
+/* Generated coding chart table */
+#include "daitch_mokotoff.h"
+
+#define DM_CODE_DIGITS 6
+
+/* Node in soundex code tree */
+struct dm_node
+{
+ int soundex_length; /* Length of generated soundex code */
+ char soundex[DM_CODE_DIGITS + 1]; /* Soundex code */
+ int is_leaf; /* Candidate for complete soundex code */
+ int last_update; /* Letter number for last update of node */
+ char code_digit; /* Last code digit, 0 - 9 */
+
+ /*
+ * One or two alternate code digits leading to this node. If there are two
+ * digits, one of them is always an 'X'. Repeated code digits and 'X' lead
+ * back to the same node.
+ */
+ char prev_code_digits[2];
+ /* One or two alternate code digits moving forward. */
+ char next_code_digits[2];
+ /* ORed together code index(es) used to reach current node. */
+ int prev_code_index;
+ int next_code_index;
+ /* Nodes branching out from this node. */
+ struct dm_node *children[DM_MAX_ALTERNATE_CODES + 1];
+ /* Next node in linked list. Alternating index for each iteration. */
+ struct dm_node *next[2];
+};
+
+typedef struct dm_node dm_node;
+
+
+/* Internal C implementation */
+static int daitch_mokotoff_coding(char *word, StringInfo soundex);
+
+
+PG_FUNCTION_INFO_V1(daitch_mokotoff);
+
+Datum
+daitch_mokotoff(PG_FUNCTION_ARGS)
+{
+ text *arg = PG_GETARG_TEXT_PP(0);
+ char *string;
+ StringInfoData soundex;
+ text *retval;
+ MemoryContext old_ctx,
+ tmp_ctx;
+
+ tmp_ctx = AllocSetContextCreate(CurrentMemoryContext,
+ "daitch_mokotoff temporary context",
+ ALLOCSET_DEFAULT_SIZES);
+ old_ctx = MemoryContextSwitchTo(tmp_ctx);
+
+ string = pg_server_to_any(text_to_cstring(arg), VARSIZE_ANY_EXHDR(arg), PG_UTF8);
+ initStringInfo(&soundex);
+
+ if (!daitch_mokotoff_coding(string, &soundex))
+ {
+ /* No encodable characters in input. */
+ MemoryContextSwitchTo(old_ctx);
+ MemoryContextDelete(tmp_ctx);
+ PG_RETURN_NULL();
+ }
+
+ string = pg_any_to_server(soundex.data, soundex.len, PG_UTF8);
+ MemoryContextSwitchTo(old_ctx);
+ retval = cstring_to_text(string);
+ MemoryContextDelete(tmp_ctx);
+
+ PG_RETURN_TEXT_P(retval);
+}
+
+
+/* Template for new node in soundex code tree. */
+static const dm_node start_node = {
+ .soundex_length = 0,
+ .soundex = "000000 ", /* Six digits + joining space */
+ .is_leaf = 0,
+ .last_update = 0,
+ .code_digit = '\0',
+ .prev_code_digits = {'\0', '\0'},
+ .next_code_digits = {'\0', '\0'},
+ .prev_code_index = 0,
+ .next_code_index = 0,
+ .children = {NULL},
+ .next = {NULL}
+};
+
+/* Dummy soundex codes at end of input. */
+static const dm_codes end_codes[2] =
+{
+ {
+ "X", "X", "X"
+ }
+};
+
+
+/* Initialize soundex code tree node for next code digit. */
+static void
+initialize_node(dm_node * node, int last_update)
+{
+ if (node->last_update < last_update)
+ {
+ node->prev_code_digits[0] = node->next_code_digits[0];
+ node->prev_code_digits[1] = node->next_code_digits[1];
+ node->next_code_digits[0] = '\0';
+ node->next_code_digits[1] = '\0';
+ node->prev_code_index = node->next_code_index;
+ node->next_code_index = 0;
+ node->is_leaf = 0;
+ node->last_update = last_update;
+ }
+}
+
+
+/* Update soundex code tree node with next code digit. */
+static void
+add_next_code_digit(dm_node * node, int code_index, char code_digit)
+{
+ /* OR in index 1 or 2. */
+ node->next_code_index |= code_index;
+
+ if (!node->next_code_digits[0])
+ {
+ node->next_code_digits[0] = code_digit;
+ }
+ else if (node->next_code_digits[0] != code_digit)
+ {
+ node->next_code_digits[1] = code_digit;
+ }
+}
+
+
+/* Mark soundex code tree node as leaf. */
+static void
+set_leaf(dm_node * first_node[2], dm_node * last_node[2], dm_node * node, int ix_node)
+{
+ if (!node->is_leaf)
+ {
+ node->is_leaf = 1;
+
+ if (first_node[ix_node] == NULL)
+ {
+ first_node[ix_node] = node;
+ }
+ else
+ {
+ last_node[ix_node]->next[ix_node] = node;
+ }
+
+ last_node[ix_node] = node;
+ node->next[ix_node] = NULL;
+ }
+}
+
+
+/* Find next node corresponding to code digit, or create a new node. */
+static dm_node *
+find_or_create_child_node(dm_node * parent, char code_digit, StringInfo soundex)
+{
+ dm_node **nodes;
+ dm_node *node;
+ int i;
+
+ for (nodes = parent->children, i = 0; (node = nodes[i]); i++)
+ {
+ if (node->code_digit == code_digit)
+ {
+ /* Found existing child node. Skip completed nodes. */
+ return node->soundex_length < DM_CODE_DIGITS ? node : NULL;
+ }
+ }
+
+ /* Create new child node. */
+ Assert(i < DM_MAX_ALTERNATE_CODES);
+ node = palloc(sizeof(dm_node));
+ nodes[i] = node;
+
+ *node = start_node;
+ memcpy(node->soundex, parent->soundex, sizeof(parent->soundex));
+ node->soundex_length = parent->soundex_length;
+ node->soundex[node->soundex_length++] = code_digit;
+ node->code_digit = code_digit;
+ node->next_code_index = node->prev_code_index;
+
+ if (node->soundex_length < DM_CODE_DIGITS)
+ {
+ return node;
+ }
+ else
+ {
+ /* Append completed soundex code to soundex string. */
+ appendBinaryStringInfoNT(soundex, node->soundex, DM_CODE_DIGITS + 1);
+ return NULL;
+ }
+}
+
+
+/* Update node for next code digit(s). */
+static void
+update_node(dm_node * first_node[2], dm_node * last_node[2], dm_node * node, int ix_node,
+ int letter_no, int prev_code_index, int next_code_index,
+ const char *next_code_digits, int digit_no,
+ StringInfo soundex)
+{
+ int i;
+ char next_code_digit = next_code_digits[digit_no];
+ int num_dirty_nodes = 0;
+ dm_node *dirty_nodes[2];
+
+ initialize_node(node, letter_no);
+
+ if (node->prev_code_index && !(node->prev_code_index & prev_code_index))
+ {
+ /*
+ * If the sound (vowel / consonant) of this letter encoding doesn't
+ * correspond to the coding index of the previous letter, we skip this
+ * letter encoding. Note that currently, only "J" can be either a
+ * vowel or a consonant.
+ */
+ return;
+ }
+
+ if (next_code_digit == 'X' ||
+ (digit_no == 0 &&
+ (node->prev_code_digits[0] == next_code_digit ||
+ node->prev_code_digits[1] == next_code_digit)))
+ {
+ /* The code digit is the same as one of the previous (i.e. not added). */
+ dirty_nodes[num_dirty_nodes++] = node;
+ }
+
+ if (next_code_digit != 'X' &&
+ (digit_no > 0 ||
+ node->prev_code_digits[0] != next_code_digit ||
+ node->prev_code_digits[1]))
+ {
+ /* The code digit is different from one of the previous (i.e. added). */
+ node = find_or_create_child_node(node, next_code_digit, soundex);
+ if (node)
+ {
+ initialize_node(node, letter_no);
+ dirty_nodes[num_dirty_nodes++] = node;
+ }
+ }
+
+ for (i = 0; i < num_dirty_nodes; i++)
+ {
+ /* Add code digit leading to the current node. */
+ add_next_code_digit(dirty_nodes[i], next_code_index, next_code_digit);
+
+ if (next_code_digits[++digit_no])
+ {
+ update_node(first_node, last_node, dirty_nodes[i], ix_node,
+ letter_no, prev_code_index, next_code_index,
+ next_code_digits, digit_no,
+ soundex);
+ }
+ else
+ {
+ /* Add incomplete leaf node to linked list. */
+ set_leaf(first_node, last_node, dirty_nodes[i], ix_node);
+ }
+ }
+}
+
+
+/* Update soundex tree leaf nodes. */
+static void
+update_leaves(dm_node * first_node[2], int *ix_node, int letter_no,
+ const dm_codes * codes, const dm_codes * next_codes,
+ StringInfo soundex)
+{
+ int i,
+ j,
+ code_index;
+ dm_node *node,
+ *last_node[2];
+ const dm_code *code,
+ *next_code;
+ int ix_node_next = (*ix_node + 1) & 1; /* Alternating index: 0, 1 */
+
+ /* Initialize for new linked list of leaves. */
+ first_node[ix_node_next] = NULL;
+ last_node[ix_node_next] = NULL;
+
+ /* Process all nodes. */
+ for (node = first_node[*ix_node]; node; node = node->next[*ix_node])
+ {
+ /* One or two alternate code sequences. */
+ for (i = 0; i < 2 && (code = codes[i]) && code[0][0]; i++)
+ {
+ /* Coding for previous letter - before vowel: 1, all other: 2 */
+ int prev_code_index = (code[0][0] > '1') + 1;
+
+ /* One or two alternate next code sequences. */
+ for (j = 0; j < 2 && (next_code = next_codes[j]) && next_code[0][0]; j++)
+ {
+ /* Determine which code to use. */
+ if (letter_no == 0)
+ {
+ /* This is the first letter. */
+ code_index = 0;
+ }
+ else if (next_code[0][0] <= '1')
+ {
+ /* The next letter is a vowel. */
+ code_index = 1;
+ }
+ else
+ {
+ /* All other cases. */
+ code_index = 2;
+ }
+
+ /* One or two sequential code digits. */
+ update_node(first_node, last_node, node, ix_node_next,
+ letter_no, prev_code_index, code_index,
+ code[code_index], 0,
+ soundex);
+ }
+ }
+ }
+
+ *ix_node = ix_node_next;
+}
+
+
+/* Mapping from ISO8859-1 to upper-case ASCII */
+static const char iso8859_1_to_ascii_upper[] =
+/*
+"`abcdefghijklmnopqrstuvwxyz{|}~ ¡¢£¤¥¦§¨©ª«¬ ®¯°±²³´µ¶·¸¹º»¼½¾¿ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖ×ØÙÚÛÜÝÞßàáâãäåæçèéêëìíîïðñòóôõö÷øùúûüýþÿ"
+*/
+"`ABCDEFGHIJKLMNOPQRSTUVWXYZ{|}~ ! ?AAAAAAECEEEEIIIIDNOOOOO*OUUUUYDSAAAAAAECEEEEIIIIDNOOOOO/OUUUUYDY";
+
+
+/* Return next character, converted from UTF-8 to uppercase ASCII. */
+static char
+read_char(unsigned char *str, int *ix)
+{
+ /* Substitute character for skipped code points. */
+ const char na = '\x1a';
+ pg_wchar c;
+
+ /* Decode UTF-8 character to ISO 10646 code point. */
+ str += *ix;
+ c = utf8_to_unicode(str);
+ *ix += pg_utf_mblen(str);
+
+ if (c >= (unsigned char) '[' && c <= (unsigned char) ']')
+ {
+ /* ASCII characters [, \, and ] are reserved for Ą, Ę, and Ţ/Ț. */
+ return na;
+ }
+ else if (c < 0x60)
+ {
+ /* Non-lowercase ASCII character. */
+ return c;
+ }
+ else if (c < 0x100)
+ {
+ /* ISO-8859-1 code point, converted to upper-case ASCII. */
+ return iso8859_1_to_ascii_upper[c - 0x60];
+ }
+ else
+ {
+ /* Conversion of non-ASCII characters in the coding chart. */
+ switch (c)
+ {
+ case 0x0104:
+ case 0x0105:
+ /* Ą/ą */
+ return '[';
+ case 0x0118:
+ case 0x0119:
+ /* Ę/ę */
+ return '\\';
+ case 0x0162:
+ case 0x0163:
+ case 0x021A:
+ case 0x021B:
+ /* Ţ/ţ or Ț/ț */
+ return ']';
+ default:
+ return na;
+ }
+ }
+}
+
+
+/* Read next ASCII character, skipping any characters not in [A-\]]. */
+static char
+read_valid_char(char *str, int *ix)
+{
+ char c;
+
+ while ((c = read_char((unsigned char *) str, ix)))
+ {
+ if (c >= 'A' && c <= ']')
+ {
+ break;
+ }
+ }
+
+ return c;
+}
+
+
+/* Return sound coding for "letter" (letter sequence) */
+static const dm_codes *
+read_letter(char *str, int *ix)
+{
+ char c,
+ cmp;
+ int i,
+ j;
+ const dm_letter *letters;
+ const dm_codes *codes;
+
+ /* First letter in sequence. */
+ if (!(c = read_valid_char(str, ix)))
+ {
+ return NULL;
+ }
+ letters = &letter_[c - 'A'];
+ codes = letters->codes;
+ i = *ix;
+
+ /* Any subsequent letters in sequence. */
+ while ((letters = letters->letters) && (c = read_valid_char(str, &i)))
+ {
+ for (j = 0; (cmp = letters[j].letter); j++)
+ {
+ if (cmp == c)
+ {
+ /* Letter found. */
+ letters = &letters[j];
+ if (letters->codes)
+ {
+ /* Coding for letter sequence found. */
+ codes = letters->codes;
+ *ix = i;
+ }
+ break;
+ }
+ }
+ if (!cmp)
+ {
+ /* The sequence of letters has no coding. */
+ break;
+ }
+ }
+
+ return codes;
+}
+
+
+/* Generate all Daitch-Mokotoff soundex codes for word, separated by space. */
+static int
+daitch_mokotoff_coding(char *word, StringInfo soundex)
+{
+ int i = 0;
+ int letter_no = 0;
+ int ix_node = 0;
+ const dm_codes *codes,
+ *next_codes;
+ dm_node *first_node[2],
+ *node;
+
+ /* First letter. */
+ if (!(codes = read_letter(word, &i)))
+ {
+ /* No encodable character in input. */
+ return 0;
+ }
+
+ /* Starting point. */
+ first_node[ix_node] = palloc(sizeof(dm_node));
+ *first_node[ix_node] = start_node;
+
+ /*
+ * Loop until either the word input is exhausted, or all generated soundex
+ * codes are completed to six digits.
+ */
+ while (codes && first_node[ix_node])
+ {
+ next_codes = read_letter(word, &i);
+
+ /* Update leaf nodes. */
+ update_leaves(first_node, &ix_node, letter_no,
+ codes, next_codes ? next_codes : end_codes,
+ soundex);
+
+ codes = next_codes;
+ letter_no++;
+ }
+
+ /* Append all remaining (incomplete) soundex codes. */
+ for (node = first_node[ix_node]; node; node = node->next[ix_node])
+ {
+ appendBinaryStringInfoNT(soundex, node->soundex, DM_CODE_DIGITS + 1);
+ }
+
+ /* Terminate string at the final space. */
+ soundex->len--;
+ soundex->data[soundex->len] = '\0';
+
+ return 1;
+}
diff --git a/contrib/fuzzystrmatch/daitch_mokotoff_header.pl b/contrib/fuzzystrmatch/daitch_mokotoff_header.pl
new file mode 100755
index 0000000000..c5eb49a6ff
--- /dev/null
+++ b/contrib/fuzzystrmatch/daitch_mokotoff_header.pl
@@ -0,0 +1,274 @@
+#!/bin/perl
+#
+# Generation of types and lookup tables for Daitch-Mokotoff soundex.
+#
+# Copyright (c) 2021 Finance Norway
+# Author: Dag Lem <[email protected]>
+#
+# Permission to use, copy, modify, and distribute this software and its
+# documentation for any purpose, without fee, and without a written agreement
+# is hereby granted, provided that the above copyright notice and this
+# paragraph and the following two paragraphs appear in all copies.
+#
+# IN NO EVENT SHALL THE AUTHOR OR DISTRIBUTORS BE LIABLE TO ANY PARTY FOR
+# DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING
+# LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS
+# DOCUMENTATION, EVEN IF THE AUTHOR OR DISTRIBUTORS HAVE BEEN ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+#
+# THE AUTHOR AND DISTRIBUTORS SPECIFICALLY DISCLAIM ANY WARRANTIES,
+# INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
+# AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS
+# ON AN "AS IS" BASIS, AND THE AUTHOR AND DISTRIBUTORS HAS NO OBLIGATIONS TO
+# PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
+#
+
+use strict;
+use warnings;
+use utf8;
+use open IO => ':utf8', ':std';
+use Data::Dumper;
+
+die "Usage: $0 OUTPUT_FILE\n" if @ARGV != 1;
+my $output_file = $ARGV[0];
+
+# Open the output file
+open my $OUTPUT, '>', $output_file
+ or die "Could not open output file $output_file: $!\n";
+
+# Parse code table and generate tree for letter transitions.
+my %codes;
+my %alternates;
+my $table = [{}, [["","",""]]];
+while (<DATA>) {
+ chomp;
+ my ($letters, $codes) = split(/\s+/);
+ my @codes = map { [ split(/,/) ] } split(/\|/, $codes);
+
+ # Find alternate code transitions for calculation of storage.
+ # The first character ("start of a name") can never yield more than two alternate codes,
+ # and is not considered here.
+ if (@codes > 1) {
+ for my $j (1..2) { # Codes for "before a vowel" and "any other"
+ for my $i (0..1) { # Alternate codes
+ # Identical code digits for adjacent letters are collapsed.
+ # For each possible non-transition due to code digit
+ # collapsing, find all alternate transitions.
+ my ($present, $next) = ($codes[$i][$j], $codes[($i + 1)%2][$j]);
+ next if length($present) != 1;
+ $next = $present ne substr($next, 0, 1) ? substr($next, 0, 1) : substr($next, -1, 1);
+ $alternates{$present}{$next} = 1;
+ }
+ }
+ }
+ my $key = "codes_" . join("_or_", map { join("_", @$_) } @codes);
+ my $val = join(",\n", map { "\t{\n\t\t" . join(", ", map { "\"$_\"" } @$_) . "\n\t}" } @codes);
+ $codes{$key} = $val;
+
+ for my $letter (split(/,/, $letters)) {
+ my $ref = $table->[0];
+ # Link each character to the next in the letter combination.
+ my @c = split(//, $letter);
+ my $last_c = pop(@c);
+ for my $c (@c) {
+ $ref->{$c} //= [ {}, undef ];
+ $ref->{$c}[0] //= {};
+ $ref = $ref->{$c}[0];
+ }
+ # The sound code for the letter combination is stored at the last character.
+ $ref->{$last_c}[1] = $key;
+ }
+}
+close(DATA);
+
+# Add alternates by following transitions to 'X' (not coded).
+my $alt_x = $alternates{"X"};
+delete $alt_x->{"X"};
+while (my ($k, $v) = each %alternates) {
+ if (delete $v->{"X"}) {
+ for my $x (keys %$alt_x) {
+ $v->{$x} = 1;
+ }
+ }
+}
+
+# Find the maximum number of alternate codes in one position.
+# Add two for any additional final code digit transitions.
+my $max_alt = (sort { $b <=> $a } (map { scalar keys %$_ } values %alternates))[0] + 2;
+
+print $OUTPUT <<EOF;
+/*
+ * Constants and lookup tables for Daitch-Mokotoff Soundex
+ *
+ * Copyright (c) 2021 Finance Norway
+ * Author: Dag Lem <dag\@nimrod.no>
+ *
+ * This file is generated by daitch_mokotoff_header.pl
+ *
+ * Permission to use, copy, modify, and distribute this software and its
+ * documentation for any purpose, without fee, and without a written agreement
+ * is hereby granted, provided that the above copyright notice and this
+ * paragraph and the following two paragraphs appear in all copies.
+ *
+ * IN NO EVENT SHALL THE AUTHOR OR DISTRIBUTORS BE LIABLE TO ANY PARTY FOR
+ * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING
+ * LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS
+ * DOCUMENTATION, EVEN IF THE AUTHOR OR DISTRIBUTORS HAVE BEEN ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ *
+ * THE AUTHOR AND DISTRIBUTORS SPECIFICALLY DISCLAIM ANY WARRANTIES,
+ * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
+ * AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS
+ * ON AN "AS IS" BASIS, AND THE AUTHOR AND DISTRIBUTORS HAS NO OBLIGATIONS TO
+ * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
+ */
+
+#define DM_MAX_ALTERNATE_CODES $max_alt
+
+/* Coding chart table: Soundex codes */
+typedef char dm_code[2 + 1]; /* One or two sequential code digits + NUL */
+typedef dm_code dm_codes[3]; /* Start of name, before a vowel, any other */
+
+/* Coding chart table: Letter in input sequence */
+struct dm_letter
+{
+ char letter; /* Present letter in sequence */
+ const struct dm_letter *letters; /* List of possible successive letters */
+ const dm_codes *codes; /* Code sequence(s) for complete sequence */
+};
+
+typedef struct dm_letter dm_letter;
+
+/* Codes for letter sequence at start of name, before a vowel, and any other. */
+EOF
+
+for my $key (sort keys %codes) {
+ print $OUTPUT "static const dm_codes $key\[2\] =\n{\n" . $codes{$key} . "\n};\n";
+}
+
+print $OUTPUT <<EOF;
+
+/* Coding for alternative following letters in sequence. */
+EOF
+
+sub hash2code {
+ my ($ref, $letter) = @_;
+
+ my @letters = ();
+
+ my $h = $ref->[0];
+ for my $key (sort keys %$h) {
+ $ref = $h->{$key};
+ my $children = "NULL";
+ if (defined $ref->[0]) {
+ $children = "letter_$letter$key";
+ hash2code($ref, "$letter$key");
+ }
+ my $codes = $ref->[1] // "NULL";
+ push(@letters, "\t{\n\t\t'$key', $children, $codes\n\t}");
+ }
+
+ print $OUTPUT "static const dm_letter letter_$letter\[\] =\n{\n";
+ for (@letters) {
+ print $OUTPUT "$_,\n";
+ }
+ print $OUTPUT "\t{\n\t\t'\\0'\n\t}\n";
+ print $OUTPUT "};\n";
+}
+
+hash2code($table, '');
+
+close $OUTPUT;
+
+# Table adapted from https://www.jewishgen.org/InfoFiles/Soundex.html
+#
+# The conversion from the coding chart to the table should be self
+# explanatory, but note the differences stated below.
+#
+# X = NC (not coded)
+#
+# The non-ASCII letters in the coding chart are coded with substitute
+# lowercase ASCII letters, which sort after the uppercase ASCII letters:
+#
+# Ą => a (use '[' for table lookup)
+# Ę => e (use '\\' for table lookup)
+# Ţ => t (use ']' for table lookup)
+#
+# The rule for "UE" does not correspond to the coding chart, however
+# it is used by all other known implementations, including the one at
+# https://www.jewishgen.org/jos/jossound.htm (try e.g. "bouey").
+#
+# Note that the implementation assumes that vowels are assigned code
+# 0 or 1. "J" can be either a vowel or a consonant.
+#
+
+__DATA__
+AI,AJ,AY 0,1,X
+AU 0,7,X
+a X,X,6|X,X,X
+A 0,X,X
+B 7,7,7
+CHS 5,54,54
+CH 5,5,5|4,4,4
+CK 5,5,5|45,45,45
+CZ,CS,CSZ,CZS 4,4,4
+C 5,5,5|4,4,4
+DRZ,DRS 4,4,4
+DS,DSH,DSZ 4,4,4
+DZ,DZH,DZS 4,4,4
+D,DT 3,3,3
+EI,EJ,EY 0,1,X
+EU 1,1,X
+e X,X,6|X,X,X
+E 0,X,X
+FB 7,7,7
+F 7,7,7
+G 5,5,5
+H 5,5,X
+IA,IE,IO,IU 1,X,X
+I 0,X,X
+J 1,X,X|4,4,4
+KS 5,54,54
+KH 5,5,5
+K 5,5,5
+L 8,8,8
+MN 66,66,66
+M 6,6,6
+NM 66,66,66
+N 6,6,6
+OI,OJ,OY 0,1,X
+O 0,X,X
+P,PF,PH 7,7,7
+Q 5,5,5
+RZ,RS 94,94,94|4,4,4
+R 9,9,9
+SCHTSCH,SCHTSH,SCHTCH 2,4,4
+SCH 4,4,4
+SHTCH,SHCH,SHTSH 2,4,4
+SHT,SCHT,SCHD 2,43,43
+SH 4,4,4
+STCH,STSCH,SC 2,4,4
+STRZ,STRS,STSH 2,4,4
+ST 2,43,43
+SZCZ,SZCS 2,4,4
+SZT,SHD,SZD,SD 2,43,43
+SZ 4,4,4
+S 4,4,4
+TCH,TTCH,TTSCH 4,4,4
+TH 3,3,3
+TRZ,TRS 4,4,4
+TSCH,TSH 4,4,4
+TS,TTS,TTSZ,TC 4,4,4
+TZ,TTZ,TZS,TSZ 4,4,4
+t 3,3,3|4,4,4
+T 3,3,3
+UI,UJ,UY,UE 0,1,X
+U 0,X,X
+V 7,7,7
+W 7,7,7
+X 5,54,54
+Y 1,X,X
+ZDZ,ZDZH,ZHDZH 2,4,4
+ZD,ZHD 2,43,43
+ZH,ZS,ZSCH,ZSH 4,4,4
+Z 4,4,4
diff --git a/contrib/fuzzystrmatch/expected/fuzzystrmatch.out b/contrib/fuzzystrmatch/expected/fuzzystrmatch.out
index 493c95cdfa..f62ddad4ee 100644
--- a/contrib/fuzzystrmatch/expected/fuzzystrmatch.out
+++ b/contrib/fuzzystrmatch/expected/fuzzystrmatch.out
@@ -65,3 +65,174 @@ SELECT dmetaphone_alt('gumbo');
KMP
(1 row)
+-- Wovels
+SELECT daitch_mokotoff('Augsburg');
+ daitch_mokotoff
+-----------------
+ 054795
+(1 row)
+
+SELECT daitch_mokotoff('Breuer');
+ daitch_mokotoff
+-----------------
+ 791900
+(1 row)
+
+SELECT daitch_mokotoff('Freud');
+ daitch_mokotoff
+-----------------
+ 793000
+(1 row)
+
+-- The letter "H"
+SELECT daitch_mokotoff('Halberstadt');
+ daitch_mokotoff
+-----------------
+ 587943 587433
+(1 row)
+
+SELECT daitch_mokotoff('Mannheim');
+ daitch_mokotoff
+-----------------
+ 665600
+(1 row)
+
+-- Adjacent sounds
+SELECT daitch_mokotoff('Chernowitz');
+ daitch_mokotoff
+-----------------
+ 596740 496740
+(1 row)
+
+-- Adjacent letters with identical adjacent code digits
+SELECT daitch_mokotoff('Cherkassy');
+ daitch_mokotoff
+-----------------
+ 595400 495400
+(1 row)
+
+SELECT daitch_mokotoff('Kleinman');
+ daitch_mokotoff
+-----------------
+ 586660
+(1 row)
+
+-- More than one word
+SELECT daitch_mokotoff('Nowy Targ');
+ daitch_mokotoff
+-----------------
+ 673950
+(1 row)
+
+-- Padded with "0"
+SELECT daitch_mokotoff('Berlin');
+ daitch_mokotoff
+-----------------
+ 798600
+(1 row)
+
+-- Other examples from https://www.avotaynu.com/soundex.htm
+SELECT daitch_mokotoff('Ceniow');
+ daitch_mokotoff
+-----------------
+ 567000 467000
+(1 row)
+
+SELECT daitch_mokotoff('Tsenyuv');
+ daitch_mokotoff
+-----------------
+ 467000
+(1 row)
+
+SELECT daitch_mokotoff('Holubica');
+ daitch_mokotoff
+-----------------
+ 587500 587400
+(1 row)
+
+SELECT daitch_mokotoff('Golubitsa');
+ daitch_mokotoff
+-----------------
+ 587400
+(1 row)
+
+SELECT daitch_mokotoff('Przemysl');
+ daitch_mokotoff
+-----------------
+ 794648 746480
+(1 row)
+
+SELECT daitch_mokotoff('Pshemeshil');
+ daitch_mokotoff
+-----------------
+ 746480
+(1 row)
+
+SELECT daitch_mokotoff('Rosochowaciec');
+ daitch_mokotoff
+---------------------------------------------------------
+ 945755 945754 945745 945744 944755 944754 944745 944744
+(1 row)
+
+SELECT daitch_mokotoff('Rosokhovatsets');
+ daitch_mokotoff
+-----------------
+ 945744
+(1 row)
+
+-- Ignored characters
+SELECT daitch_mokotoff('''OBrien');
+ daitch_mokotoff
+-----------------
+ 079600
+(1 row)
+
+SELECT daitch_mokotoff('O''Brien');
+ daitch_mokotoff
+-----------------
+ 079600
+(1 row)
+
+-- "Difficult" cases, likely to cause trouble for other implementations.
+SELECT daitch_mokotoff('CJC');
+ daitch_mokotoff
+-------------------------------------------
+ 550000 540000 545000 450000 400000 440000
+(1 row)
+
+SELECT daitch_mokotoff('BESST');
+ daitch_mokotoff
+-----------------
+ 743000
+(1 row)
+
+SELECT daitch_mokotoff('BOUEY');
+ daitch_mokotoff
+-----------------
+ 710000
+(1 row)
+
+SELECT daitch_mokotoff('HANNMANN');
+ daitch_mokotoff
+-----------------
+ 566600
+(1 row)
+
+SELECT daitch_mokotoff('MCCOYJR');
+ daitch_mokotoff
+---------------------------------------------------------
+ 651900 654900 654190 654490 645190 645490 641900 644900
+(1 row)
+
+SELECT daitch_mokotoff('ACCURSO');
+ daitch_mokotoff
+---------------------------------------------------------
+ 059400 054000 054940 054400 045940 045400 049400 044000
+(1 row)
+
+SELECT daitch_mokotoff('BIERSCHBACH');
+ daitch_mokotoff
+---------------------------------------------------------
+ 794575 794574 794750 794740 745750 745740 747500 747400
+(1 row)
+
diff --git a/contrib/fuzzystrmatch/expected/fuzzystrmatch_utf8.out b/contrib/fuzzystrmatch/expected/fuzzystrmatch_utf8.out
new file mode 100644
index 0000000000..32d8260383
--- /dev/null
+++ b/contrib/fuzzystrmatch/expected/fuzzystrmatch_utf8.out
@@ -0,0 +1,61 @@
+/*
+ * This test must be run in a database with UTF-8 encoding,
+ * because other encodings don't support all the characters used.
+ */
+SELECT getdatabaseencoding() <> 'UTF8'
+ AS skip_test \gset
+\if :skip_test
+\quit
+\endif
+set client_encoding = utf8;
+-- CREATE EXTENSION IF NOT EXISTS fuzzystrmatch;
+-- Accents
+SELECT daitch_mokotoff('Müller');
+ daitch_mokotoff
+-----------------
+ 689000
+(1 row)
+
+SELECT daitch_mokotoff('Schäfer');
+ daitch_mokotoff
+-----------------
+ 479000
+(1 row)
+
+SELECT daitch_mokotoff('Straßburg');
+ daitch_mokotoff
+-----------------
+ 294795
+(1 row)
+
+SELECT daitch_mokotoff('Éregon');
+ daitch_mokotoff
+-----------------
+ 095600
+(1 row)
+
+-- Special characters added at https://www.jewishgen.org/InfoFiles/Soundex.html
+SELECT daitch_mokotoff('gąszczu');
+ daitch_mokotoff
+-----------------
+ 564000 540000
+(1 row)
+
+SELECT daitch_mokotoff('brzęczy');
+ daitch_mokotoff
+-----------------------------
+ 794640 794400 746400 744000
+(1 row)
+
+SELECT daitch_mokotoff('ţamas');
+ daitch_mokotoff
+-----------------
+ 364000 464000
+(1 row)
+
+SELECT daitch_mokotoff('țamas');
+ daitch_mokotoff
+-----------------
+ 364000 464000
+(1 row)
+
diff --git a/contrib/fuzzystrmatch/expected/fuzzystrmatch_utf8_1.out b/contrib/fuzzystrmatch/expected/fuzzystrmatch_utf8_1.out
new file mode 100644
index 0000000000..37aead89c0
--- /dev/null
+++ b/contrib/fuzzystrmatch/expected/fuzzystrmatch_utf8_1.out
@@ -0,0 +1,8 @@
+/*
+ * This test must be run in a database with UTF-8 encoding,
+ * because other encodings don't support all the characters used.
+ */
+SELECT getdatabaseencoding() <> 'UTF8'
+ AS skip_test \gset
+\if :skip_test
+\quit
diff --git a/contrib/fuzzystrmatch/fuzzystrmatch--1.1--1.2.sql b/contrib/fuzzystrmatch/fuzzystrmatch--1.1--1.2.sql
new file mode 100644
index 0000000000..b9d7b229a3
--- /dev/null
+++ b/contrib/fuzzystrmatch/fuzzystrmatch--1.1--1.2.sql
@@ -0,0 +1,8 @@
+/* contrib/fuzzystrmatch/fuzzystrmatch--1.1--1.2.sql */
+
+-- complain if script is sourced in psql, rather than via ALTER EXTENSION
+\echo Use "ALTER EXTENSION fuzzystrmatch UPDATE TO '1.2'" to load this file. \quit
+
+CREATE FUNCTION daitch_mokotoff(text) RETURNS text
+AS 'MODULE_PATHNAME', 'daitch_mokotoff'
+LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE;
diff --git a/contrib/fuzzystrmatch/fuzzystrmatch.control b/contrib/fuzzystrmatch/fuzzystrmatch.control
index 3cd6660bf9..8b6e9fd993 100644
--- a/contrib/fuzzystrmatch/fuzzystrmatch.control
+++ b/contrib/fuzzystrmatch/fuzzystrmatch.control
@@ -1,6 +1,6 @@
# fuzzystrmatch extension
comment = 'determine similarities and distance between strings'
-default_version = '1.1'
+default_version = '1.2'
module_pathname = '$libdir/fuzzystrmatch'
relocatable = true
trusted = true
diff --git a/contrib/fuzzystrmatch/meson.build b/contrib/fuzzystrmatch/meson.build
index 11aec733cb..abb944faaf 100644
--- a/contrib/fuzzystrmatch/meson.build
+++ b/contrib/fuzzystrmatch/meson.build
@@ -1,9 +1,18 @@
# Copyright (c) 2022, PostgreSQL Global Development Group
fuzzystrmatch_sources = files(
- 'fuzzystrmatch.c',
+ 'daitch_mokotoff.c',
'dmetaphone.c',
+ 'fuzzystrmatch.c',
+)
+
+daitch_mokotoff_h = custom_target('daitch_mokotoff',
+ input: 'daitch_mokotoff_header.pl',
+ output: 'daitch_mokotoff.h',
+ command: [perl, '@INPUT@', '@OUTPUT@'],
)
+generated_sources += daitch_mokotoff_h
+fuzzystrmatch_sources += daitch_mokotoff_h
if host_system == 'windows'
fuzzystrmatch_sources += rc_lib_gen.process(win32ver_rc, extra_args: [
@@ -13,6 +22,7 @@ endif
fuzzystrmatch = shared_module('fuzzystrmatch',
fuzzystrmatch_sources,
+ include_directories: include_directories('.'),
kwargs: contrib_mod_args,
)
contrib_targets += fuzzystrmatch
@@ -21,6 +31,7 @@ install_data(
'fuzzystrmatch.control',
'fuzzystrmatch--1.0--1.1.sql',
'fuzzystrmatch--1.1.sql',
+ 'fuzzystrmatch--1.1--1.2.sql',
kwargs: contrib_data_args,
)
@@ -31,6 +42,7 @@ tests += {
'regress': {
'sql': [
'fuzzystrmatch',
+ 'fuzzystrmatch_utf8',
],
},
}
diff --git a/contrib/fuzzystrmatch/sql/fuzzystrmatch.sql b/contrib/fuzzystrmatch/sql/fuzzystrmatch.sql
index f05dc28ffb..db05c7d6b6 100644
--- a/contrib/fuzzystrmatch/sql/fuzzystrmatch.sql
+++ b/contrib/fuzzystrmatch/sql/fuzzystrmatch.sql
@@ -19,3 +19,48 @@ SELECT metaphone('GUMBO', 4);
SELECT dmetaphone('gumbo');
SELECT dmetaphone_alt('gumbo');
+
+-- Wovels
+SELECT daitch_mokotoff('Augsburg');
+SELECT daitch_mokotoff('Breuer');
+SELECT daitch_mokotoff('Freud');
+
+-- The letter "H"
+SELECT daitch_mokotoff('Halberstadt');
+SELECT daitch_mokotoff('Mannheim');
+
+-- Adjacent sounds
+SELECT daitch_mokotoff('Chernowitz');
+
+-- Adjacent letters with identical adjacent code digits
+SELECT daitch_mokotoff('Cherkassy');
+SELECT daitch_mokotoff('Kleinman');
+
+-- More than one word
+SELECT daitch_mokotoff('Nowy Targ');
+
+-- Padded with "0"
+SELECT daitch_mokotoff('Berlin');
+
+-- Other examples from https://www.avotaynu.com/soundex.htm
+SELECT daitch_mokotoff('Ceniow');
+SELECT daitch_mokotoff('Tsenyuv');
+SELECT daitch_mokotoff('Holubica');
+SELECT daitch_mokotoff('Golubitsa');
+SELECT daitch_mokotoff('Przemysl');
+SELECT daitch_mokotoff('Pshemeshil');
+SELECT daitch_mokotoff('Rosochowaciec');
+SELECT daitch_mokotoff('Rosokhovatsets');
+
+-- Ignored characters
+SELECT daitch_mokotoff('''OBrien');
+SELECT daitch_mokotoff('O''Brien');
+
+-- "Difficult" cases, likely to cause trouble for other implementations.
+SELECT daitch_mokotoff('CJC');
+SELECT daitch_mokotoff('BESST');
+SELECT daitch_mokotoff('BOUEY');
+SELECT daitch_mokotoff('HANNMANN');
+SELECT daitch_mokotoff('MCCOYJR');
+SELECT daitch_mokotoff('ACCURSO');
+SELECT daitch_mokotoff('BIERSCHBACH');
diff --git a/contrib/fuzzystrmatch/sql/fuzzystrmatch_utf8.sql b/contrib/fuzzystrmatch/sql/fuzzystrmatch_utf8.sql
new file mode 100644
index 0000000000..f42c01a1bb
--- /dev/null
+++ b/contrib/fuzzystrmatch/sql/fuzzystrmatch_utf8.sql
@@ -0,0 +1,26 @@
+/*
+ * This test must be run in a database with UTF-8 encoding,
+ * because other encodings don't support all the characters used.
+ */
+
+SELECT getdatabaseencoding() <> 'UTF8'
+ AS skip_test \gset
+\if :skip_test
+\quit
+\endif
+
+set client_encoding = utf8;
+
+-- CREATE EXTENSION IF NOT EXISTS fuzzystrmatch;
+
+-- Accents
+SELECT daitch_mokotoff('Müller');
+SELECT daitch_mokotoff('Schäfer');
+SELECT daitch_mokotoff('Straßburg');
+SELECT daitch_mokotoff('Éregon');
+
+-- Special characters added at https://www.jewishgen.org/InfoFiles/Soundex.html
+SELECT daitch_mokotoff('gąszczu');
+SELECT daitch_mokotoff('brzęczy');
+SELECT daitch_mokotoff('ţamas');
+SELECT daitch_mokotoff('țamas');
diff --git a/doc/src/sgml/fuzzystrmatch.sgml b/doc/src/sgml/fuzzystrmatch.sgml
index 382e54be91..08781778f8 100644
--- a/doc/src/sgml/fuzzystrmatch.sgml
+++ b/doc/src/sgml/fuzzystrmatch.sgml
@@ -241,4 +241,101 @@ test=# SELECT dmetaphone('gumbo');
</screen>
</sect2>
+ <sect2>
+ <title>Daitch-Mokotoff Soundex</title>
+
+ <para>
+ Compared to the American Soundex System implemented in the
+ <function>soundex</function> function, the major improvements of the
+ Daitch-Mokotoff Soundex System are:
+
+ <itemizedlist spacing="compact" mark="bullet">
+ <listitem>
+ <para>
+ Information is coded to the first six meaningful letters rather than
+ four.
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ The initial letter is coded rather than kept as is.
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ Where two consecutive letters have a single sound, they are coded as a
+ single number.
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ When a letter or combination of letters may have two different sounds,
+ it is double coded under the two different codes.
+ </para>
+ </listitem>
+ <listitem>
+ <para>
+ A letter or combination of letters maps into ten possible codes rather
+ than seven.
+ </para>
+ </listitem>
+ </itemizedlist>
+ </para>
+
+ <indexterm>
+ <primary>daitch_mokotoff</primary>
+ </indexterm>
+
+ <para>
+ The following function generates Daitch-Mokotoff soundex codes for matching
+ of similar-sounding input:
+ </para>
+
+<synopsis>
+daitch_mokotoff(text source) returns text
+</synopsis>
+
+ <para>
+ Since a Daitch-Mokotoff soundex code consists of only 6 digits,
+ <literal>source</literal> should be preferably a single word or name.
+ Any alternative soundex codes are separated by space, which makes the returned
+ text suited for use in Full Text Search, see <xref linkend="textsearch"/> and
+ <xref linkend="functions-textsearch"/>.
+ </para>
+
+ <para>
+ Example:
+ </para>
+
+<programlisting>
+CREATE OR REPLACE FUNCTION soundex_name(v_name text) RETURNS text AS $$
+ SELECT string_agg(daitch_mokotoff(n), ' ')
+ FROM regexp_split_to_table(v_name, '\s+') AS n
+$$ LANGUAGE sql STRICT IMMUTABLE PARALLEL SAFE;
+
+CREATE OR REPLACE FUNCTION soundex_tsvector(v_name text) RETURNS tsvector AS $$
+ SELECT to_tsvector('simple', soundex_name(v_name))
+$$ LANGUAGE sql STRICT IMMUTABLE PARALLEL SAFE;
+
+CREATE OR REPLACE FUNCTION soundex_tsquery(v_name text) RETURNS tsquery AS $$
+ SELECT to_tsquery('simple', quote_literal(soundex_name(v_name)))
+$$ LANGUAGE sql STRICT IMMUTABLE PARALLEL SAFE;
+
+-- Note that searches could be more efficient with the tsvector in a separate column
+-- (no recalculation on table row recheck).
+CREATE TABLE s (nm text);
+CREATE INDEX ix_s_txt ON s USING gin (soundex_tsvector(nm)) WITH (fastupdate = off);
+
+INSERT INTO s VALUES ('John Doe');
+INSERT INTO s VALUES ('Jane Roe');
+INSERT INTO s VALUES ('Public John Q.');
+INSERT INTO s VALUES ('George Best');
+
+SELECT * FROM s WHERE soundex_tsvector(nm) @@ soundex_tsquery('john');
+SELECT * FROM s WHERE soundex_tsvector(nm) @@ soundex_tsquery('jane doe');
+SELECT * FROM s WHERE soundex_tsvector(nm) @@ soundex_tsquery('john public');
+SELECT * FROM s WHERE soundex_tsvector(nm) @@ soundex_tsquery('besst, giorgio');
+</programlisting>
+ </sect2>
+
</sect1>
^ permalink raw reply [nested|flat] 19+ messages in thread
end of thread, other threads:[~2022-12-23 20:55 UTC | newest]
Thread overview: 19+ messages (download: mbox mbox.gz follow: Atom feed)
-- links below jump to the message on this page --
2021-02-02 00:57 [PATCH 7/9] Remove the special batch mode, use a larger buffer always Tomas Vondra <[email protected]>
2022-01-03 19:16 Re: daitch_mokotoff module Andres Freund <[email protected]>
2022-01-04 13:49 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
2022-01-04 20:18 ` Re: daitch_mokotoff module Thomas Munro <[email protected]>
2022-01-05 07:05 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
2022-01-05 20:08 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
2022-02-03 14:27 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
2022-11-30 12:56 ` Re: daitch_mokotoff module Ian Lawrence Barwick <[email protected]>
2022-12-05 13:24 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
2022-12-22 14:02 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
2022-12-07 18:56 ` Re: daitch_mokotoff module Andres Freund <[email protected]>
2022-12-21 09:26 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
2022-12-22 12:00 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
2022-12-22 13:27 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
2022-12-23 11:22 ` Re: daitch_mokotoff module Andres Freund <[email protected]>
2022-12-23 20:34 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
2022-12-23 12:59 ` Re: daitch_mokotoff module Alvaro Herrera <[email protected]>
2022-12-23 14:57 ` Re: daitch_mokotoff module Tom Lane <[email protected]>
2022-12-23 20:55 ` Re: daitch_mokotoff module Dag Lem <[email protected]>
This inbox is served by agora; see mirroring instructions
for how to clone and mirror all data and code used for this inbox