-/************************************************************************/
-/* C Implementation: public.search_normalize public.naco_normalize
- *
- * Descritption:
- * This file implement public.search_normalize and public.naco_normalize.
- * These two functions is included in the PostgreSQL extension c_functions.
- * ICU4C and postgres lib is needed to build this file.
- *
- * Author: Swenyu Duan <dsy88@sina.com>, (C) 2012
- *
- * Copyright: See COPYING file that comes with this distribution.
- */
-/************************************************************************/
-#include "postgres.h"
-#include "fmgr.h"
-#include "unicode/unorm2.h"
-#include "unicode/utypes.h"
-#include "unicode/ustring.h"
-#include "unicode/uregex.h"
-#include "unicode/umachine.h"
-
-#ifdef PG_MODULE_MAGIC
-PG_MODULE_MAGIC;
-#endif
-
-static int32_t regexp_expand_string(const UChar *src,
- int32_t src_len,
- UChar *des,
- int32_t des_capacity)
-{
- int des_len;
- UChar s, e;
- const UChar *cur_pos;
- const UChar *pre_pos;
-
- if (src == NULL)
- {
- return 0;
- }
- pre_pos = src;
- cur_pos = u_strchr(src, L'-');
- des_len = 0;
-
- while (cur_pos != NULL)
- {
- if (cur_pos + 1 > src + src_len)
- {
- //Error!
- //The pattern is end with a '-'.
- return -1;
- }
-
- while (pre_pos < cur_pos - 1)
- {
- if (des != NULL)
- {
- des[des_len] = *pre_pos;
- }
- des_len++;
- pre_pos++;
- }
-
- cur_pos ++;
-
- for(s = *pre_pos; s < *cur_pos; s++)
- {
- if (des != NULL)
- {
- des[des_len] = s;
- }
- des_len++;
- }
-
- pre_pos = cur_pos;
-
- cur_pos = u_strchr(cur_pos, L'-');
- }
-
- return des_len;
-}
-
-static int32_t regexp_transliterate(const UChar *search_list,
- int32_t search_list_len,
- const UChar *replacement_list,
- int32_t replacement_list_len,
- UChar *src,
- int32_t src_len,
- UChar *des,
- int32_t des_capacity)
-{
- int i, j;
- int32_t des_len;
- UChar *cur_pos;
-
- if (search_list == NULL ||
- replacement_list == NULL)
- {
- return 0;
- }
-
- des_len = 0;
-
- for (i = 0; i < src_len; i++)
- {
- if (des != NULL)
- {
- des[des_len] = src[i];
- }
- des_len++;
-
- if(cur_pos = u_strchr(search_list, src[i]))
- {
- if(cur_pos - search_list > replacement_list_len)
- des_len--;
- }
- }
- if (des == NULL || des_len == 0)
- {
- return des_len;
- }
-
-
- for (i = 0; i < replacement_list_len; i++)
- {
- cur_pos = u_strchr(des, search_list[i]);
-
- while (cur_pos != NULL)
- {
- *cur_pos = replacement_list[i];
-
- //In case cur_pos is the last char in des.
- if (cur_pos >= des + des_len)
- {
- break;
- }
- cur_pos = u_strchr(cur_pos + 1, search_list[i]);
- }
- }
-
- return des_len;
-
-}
-
-static int32_t regexp_replace(const UChar *regexp,
- int32_t regexp_len,
- const UChar *replacement,
- int32_t replacement_len,
- UChar *src,
- int32_t src_len,
- UChar *des,
- int32_t des_capacity,
- int is_global)
-{
- URegularExpression *regular_exp;
- UErrorCode status;
- UParseError pe;
- int32_t len;
-
- if (regexp == NULL || replacement == NULL || src == NULL)
- {
- return 0;
- }
-
- status = 0;
-
-
- regular_exp = uregex_open(regexp, regexp_len, 0, &pe, &status);
- if (regular_exp == NULL)
- {
- return 0;
- }
-
- status = 0;
- uregex_setText(regular_exp, src, src_len, &status);
-
- status = 0;
- if (is_global > 0)
- {
- len = uregex_replaceAll(regular_exp,
- replacement,
- replacement_len,
- NULL,
- 0,
- &status);
- if (des == NULL || des_capacity < len)
- {
- uregex_close(regular_exp);
- return len;
- }
- status = 0;
-
- uregex_replaceAll(regular_exp,
- replacement,
- replacement_len,
- des,
- des_capacity,
- &status);
- }
- else
- {
- len = uregex_replaceFirst(regular_exp,
- replacement,
- replacement_len,
- NULL,
- 0,
- &status);
- if (des == NULL || des_capacity < len)
- {
- uregex_close(regular_exp);
- return len;
- }
-
- status = 0;
- uregex_replaceFirst(regular_exp,
- replacement,
- replacement_len,
- des,
- des_capacity,
- &status);
- }
-
- uregex_close(regular_exp);
- return len;
-}
-
-static UChar *u_strtransliterate(UChar *search_list,
- UChar *replacement_list,
- UChar *str,
- int32_t str_capacity)
-{
- int32_t search_list_len, replacement_list_len, str_len;
- UChar *des;
- int32_t des_len;
-
- if (search_list == NULL || replacement_list == NULL || str == NULL)
- {
- return NULL;
- }
-
- search_list_len = u_strlen(search_list);
- replacement_list_len = u_strlen(replacement_list);
- str_len = u_strlen(str);
-
- des_len = regexp_transliterate(search_list,
- search_list_len,
- replacement_list,
- replacement_list_len,
- str,
- str_len,
- NULL,
- 0);
- des = palloc((des_len + 1) * sizeof(UChar));
- des_len = regexp_transliterate(search_list,
- search_list_len,
- replacement_list,
- replacement_list_len,
- str,
- str_len,
- des,
- des_len);
- if(des != NULL)
- des[des_len] = '\0';
-
- pfree(str);
- return des;
-}
-
-static UChar *u_strreplace(UChar *regexp,
- UChar *replacement,
- UChar *str,
- int32_t str_capacity,
- int is_global)
-{
- int32_t regexp_len, replacement_len, str_len;
- UChar *des;
- int32_t des_len;
-
- if (regexp == NULL || replacement == NULL || str == NULL)
- {
- return NULL;
- }
-
- regexp_len = u_strlen(regexp);
- replacement_len = u_strlen(replacement);
- str_len = u_strlen(str);
-
- des_len = regexp_replace(regexp,
- regexp_len,
- replacement,
- replacement_len,
- str,
- str_len,
- NULL,
- 0,
- is_global);
-
- des = palloc((des_len + 1)* sizeof(UChar));
-
- des_len = regexp_replace(regexp,
- regexp_len,
- replacement,
- replacement_len,
- str,
- str_len,
- des,
- des_len,
- is_global);
- if(des != NULL)
- des[des_len] = '\0';
-
- pfree(str);
- return des;
-}
-
-static UChar *additional_substitutions(UChar *nustr, int is_search)
-{
- UChar uregexp[200], replacement[200];
-
- if (nustr == NULL)
- {
- return NULL;
- }
-
- u_uastrncpy(uregexp, "\\x{00C6}", sizeof("\\x{00C6}"));
- u_uastrncpy(replacement, "AE", sizeof("AE"));
-
- nustr = u_strreplace(uregexp, replacement, nustr, u_strlen(nustr), 1);
-
- u_uastrncpy(uregexp, "\\x{00DE}", sizeof("\\x{00DE}"));
- u_uastrncpy(replacement, "TH", sizeof("TH"));
-
- nustr = u_strreplace(uregexp, replacement, nustr, u_strlen(nustr), 1);
-
- u_uastrncpy(uregexp, "\\x{0152}", sizeof("\\x{0152}"));
- u_uastrncpy(replacement, "OE", sizeof("OE"));
-
- nustr = u_strreplace(uregexp, replacement, nustr, u_strlen(nustr), 1);
-
- if (is_search)
- {
- u_strncpy(uregexp,
- L"\x0110\x00D0\x00D8\x0141\x2113\x02BB\x02BC][",
- sizeof(L"\x0110\x00D0\x00D8\x0141\x2113\x02BB"
- L"\x02BC][") / sizeof(UChar));
-
- }
- else
- {
- u_strncpy(uregexp,
- L"\x0110\x00D0\x00D8\x0141\x2113\x02BB\x02BC]['",
- sizeof(L"\x0110\x00D0\x00D8\x0141\x2113"
- L"\x02BB\x02BC]['") / sizeof(UChar));
-
- }
- u_uastrncpy(replacement, "DDOLl", sizeof("DDOLl"));
-
- nustr = u_strtransliterate(uregexp, replacement, nustr, u_strlen(nustr));
-
- return nustr;
-}
-
-static UChar *transformations_on_unicode(UChar *nustr, UChar *usf)
-{
- UChar uregexp[200], replacement[200];
- UChar *comma;
- int32_t nustr_len;
-
- if (nustr == NULL)
- {
- return NULL;
- }
-
- nustr_len = u_strlen(nustr);
-
- u_uastrncpy(uregexp,
- "[\\p{Cc}\\p{Cf}\\p{Co}\\p{Cs}\\p{Lm}\\p{Mc}\\p{Me}\\p{Mn}]",
- sizeof("[\\p{Cc}\\p{Cf}\\p{Co}\\p{Cs}\\p{Lm}\\p{Mc}\\p{Me}\\p{Mn}]"));
- u_uastrncpy(replacement, "", sizeof(""));
-
- nustr = u_strreplace(uregexp, replacement, nustr, u_strlen(nustr), 1);
-
- if (usf != NULL && usf[0] == 0x61) //0x61 == 'a' in utf16
- {
- comma = u_strchr(nustr, 0x2c); //0x2c == ',' in utf16
- if (comma != NULL)
- {
- if (comma != nustr + nustr_len - 1)
- {
- u_uastrncpy(uregexp, ",", sizeof(","));
- replacement[0] = 0x7;
- replacement[1] = 0;
-
- nustr = u_strreplace(uregexp, replacement, nustr, u_strlen(nustr), 0);
- }
- }
- }
-
- return nustr;
-}
-
-static UChar *replace_placehoders(UChar *nustr)
-{
- UChar uregexp[200], replacement[200];
-
- if (nustr == NULL)
- {
- return NULL;
- }
-
- u_strncpy(uregexp,
- L"+&@\x266D\x266F#",
- sizeof(L"+&@\x266D\x266F#") / sizeof(UChar));
- u_strncpy(replacement,
- L"\x01\x02\x03\x04\x05\x06",
- sizeof(L"\x01\x02\x03\x04\x05\x06") / sizeof(UChar));
-
- nustr = u_strtransliterate(uregexp, replacement, nustr, u_strlen(nustr));
-
- u_uastrncpy(uregexp,
- "[\\p{Pc}\\p{Pd}\\p{Pe}\\p{Pf}\\p{Pi}\\p{Po}\\p{Ps}\\p{Sk}\\p{Sm}\\p{So}\\p{Zl}\\p{Zp}\\p{Zs}]",
- sizeof("[\\p{Pc}\\p{Pd}\\p{Pe}\\p{Pf}\\p{Pi}\\p{Po}\\p{Ps}\\p{Sk}\\p{Sm}\\p{So}\\p{Zl}\\p{Zp}\\p{Zs}]"));
- u_uastrncpy(replacement, " ", sizeof(" "));
-
- nustr = u_strreplace(uregexp, replacement, nustr, u_strlen(nustr), 1);
-
- u_strncpy(uregexp,
- L"\x01\x02\x03\x04\x05\x06\x07",
- sizeof(L"\x01\x02\x03\x04\x05\x06\x07") / sizeof(UChar));
- u_strncpy(replacement,
- L"+&@\x266D\x266F#,",
- sizeof(L"+&@\x266D\x266F#,") / sizeof(UChar));
-
- nustr = u_strtransliterate(uregexp, replacement, nustr, u_strlen(nustr));
-
-
- return nustr;
-}
-
-
-static UChar *decimal_digits(UChar *nustr)
-{
- UChar *uregexp, *replacement;
- UChar *expand_exp, *expand_replacement;
- int32_t uregexp_len, replacement_len;
- int32_t expand_exp_len, expand_replacement_len;
-
- if (nustr == NULL)
- {
- return NULL;
- }
-
- uregexp = L"\x0660-\x0669\x06F0-\x06F9\x07C0-\x07C9\x0966-\x096F"
- L"\x09E6-\x09EF\x0A66-\x0A6F\x0AE6-\x0AEF\x0B66-\x0B6F"
- L"\x0BE6-\x0BEF\x0C66-\x0C6F\x0CE6-\x0CEF\x0D66-\x0D6F"
- L"\x0E50-\x0E59\x0ED0-\x0ED9\x0F20-\x0F29\x1040-\x1049"
- L"\x1090-\x1099\x17E0-\x17E9\x1810-\x1819\x1946-\x194F"
- L"\x19D0-\x19D9\x1A80-\x1A89\x1A90-\x1A99\x1B50-\x1B59"
- L"\x1BB0-\x1BB9\x1C40-\x1C49\x1C50-\x1C59\xA620-\xA629"
- L"\xA8D0-\xA8D9\xA900-\xA909\xA9D0-\xA9D9\xAA50-\xAA59"
- L"\xABF0-\xABF9\xFF10-\xFF19";
-
- uregexp_len = sizeof(L"\x0660-\x0669\x06F0-\x06F9\x07C0-\x07C9\x0966-\x096F"
- L"\x09E6-\x09EF\x0A66-\x0A6F\x0AE6-\x0AEF\x0B66-\x0B6F"
- L"\x0BE6-\x0BEF\x0C66-\x0C6F\x0CE6-\x0CEF\x0D66-\x0D6F"
- L"\x0E50-\x0E59\x0ED0-\x0ED9\x0F20-\x0F29\x1040-\x1049"
- L"\x1090-\x1099\x17E0-\x17E9\x1810-\x1819\x1946-\x194F"
- L"\x19D0-\x19D9\x1A80-\x1A89\x1A90-\x1A99\x1B50-\x1B59"
- L"\x1BB0-\x1BB9\x1C40-\x1C49\x1C50-\x1C59\xA620-\xA629"
- L"\xA8D0-\xA8D9\xA900-\xA909\xA9D0-\xA9D9\xAA50-\xAA59"
- L"\xABF0-\xABF9\xFF10-\xFF19") / sizeof(UChar);
- replacement = L"0-90-90-90-90-90-90-90-90-90-90-90-90-90-90-90-90-90-90-90-"
- L"90-90-90-90-90-90-90-90-90-90-90-90-90-90-9";
-
- replacement_len = sizeof(L"0-90-90-90-90-90-90-90-90-90-90-90-90-90-90-90-"
- L"90-90-90-90-90-90-90-90-90-90-90-90-90-90-90-90-"
- L"90-90-9") / sizeof(UChar);
-
- expand_exp_len = regexp_expand_string(uregexp, uregexp_len, NULL, 0);
-
- expand_exp = palloc(sizeof(UChar) * (expand_exp_len + 1));
-
- expand_replacement_len = regexp_expand_string(replacement, replacement_len, NULL, 0);
-
- expand_replacement = palloc(sizeof(UChar) * (expand_replacement_len + 1));
-
- regexp_expand_string(uregexp, uregexp_len, expand_exp, expand_exp_len);
- regexp_expand_string(replacement, replacement_len, expand_replacement, expand_replacement_len);
-
- expand_exp[expand_exp_len] = '\0';
- expand_replacement[expand_replacement_len] = '\0';
-
- nustr = u_strtransliterate(expand_exp, expand_replacement, nustr, u_strlen(nustr));
-
- pfree(expand_exp);
- pfree(expand_replacement);
- return nustr;
-}
-
-static UChar *leading_trailing_spaces(UChar * nustr)
-{
- UChar uregexp[20], replacement[20];
-
- if (nustr == NULL)
- {
- return NULL;
- }
-
- u_uastrncpy(uregexp, "\\s+", sizeof("\\s+"));
- u_uastrncpy(replacement, " ", sizeof(" "));
-
- nustr = u_strreplace(uregexp, replacement, nustr, u_strlen(nustr), 1);
-
- u_uastrncpy(uregexp, "^\\s+", sizeof("^\\s+"));
- u_uastrncpy(replacement, "", sizeof(""));
- nustr = u_strreplace(uregexp, replacement, nustr, u_strlen(nustr), 0);
-
- u_uastrncpy(uregexp, "\\s+$", sizeof("\\s+$"));
-
- nustr = u_strreplace(uregexp, replacement, nustr, u_strlen(nustr), 1);
-
- return nustr;
-
-}
-
-text *normalize(text *str, text *sf, int is_search)
-{
- UChar *ustr = NULL, *nustr = NULL, *temp = NULL, *usf = NULL;
- int32_t nustr_len, temp_len, str_len, sf_len, ustr_len;
- UNormalizer2 *normalizer;
- char *regexp, *result;
- char *s;
- UChar uregexp[200], replacement[200];
- UErrorCode err = 0;
-
- if (str == NULL || sf == NULL)
- {
- return NULL;
- }
-
- normalizer = unorm2_getNFKDInstance(&err);
- if (U_FAILURE(err))
- {
- return NULL;
- }
-
- s = VARDATA(str);
-
- str_len = VARSIZE(str) - VARHDRSZ;
- sf_len = VARSIZE(sf) - VARHDRSZ;
- temp = palloc((str_len + 1)* sizeof(UChar));
- usf = palloc((sf_len + 1) * sizeof(UChar));
-
-
- temp = u_strFromUTF8(temp, str_len, NULL, s, str_len, &err);
- if (U_FAILURE(err) || temp == NULL)
- {
- str = NULL;
- goto Fail;
- }
-
- usf = u_strFromUTF8(usf, sf_len, NULL, VARDATA(sf), sf_len, &err);
- if (usf == NULL)
- {
- str = NULL;
- goto Fail;
- }
-
- usf[sf_len] = '\0';
- //Apply NACO normalization to input string; based on
- //http://www.loc.gov/catdir/pcc/naco/SCA_PccNormalization_Final_revised.pdf
- //
- //Note that unlike a strict reading of the NACO normalization rules,
- //output is returned as lowercase instead of uppercase for compatibility
- //with previous versions of the Evergreen naco_normalize routine.
- //
- //Convert to upper-case first; even though final output will be lowercase, doing this will
- //ensure that the German eszett (?) and certain ligatures (?, ?, ?, etc.) will be handled correctly.
- //If there are any bugs in Perl's implementation of upcasing, they will be passed through here.
-
- ustr_len = u_strToUpper(NULL,
- 0,
- temp,
- str_len,
- NULL,
- &err);
-
- err = 0;
- ustr = palloc((ustr_len + 1) * sizeof(UChar));
- u_strToUpper(ustr,
- ustr_len,
- temp,
- str_len,
- NULL,
- &err);
- pfree(temp);
- temp = NULL;
- ustr[ustr_len] = '\0';
- if (U_FAILURE(err))
- {
- str = NULL;
- goto Fail;
- }
-
- u_uastrncpy(uregexp,
- "\\x{0098}.*?\\x{009C}",
- sizeof("\\x{0098}.*?\\x{009C}"));
- u_uastrncpy(replacement, "", sizeof(""));
-
- ustr = u_strreplace(uregexp, replacement, ustr, u_strlen(ustr), 1);
- ustr_len = u_strlen(ustr);
-
- nustr_len = unorm2_normalize(normalizer, ustr, ustr_len, NULL, 0, &err);
-
- //To store the '\0';
-
- nustr = palloc((nustr_len + 1)* sizeof(UChar));
- err = 0;
-
- unorm2_normalize(normalizer, ustr, ustr_len, nustr, nustr_len, &err);
- if (U_FAILURE(err))
- {
- str = NULL;
- goto Fail;
- }
- nustr[nustr_len] = '\0';
-
- //additional substitutions - 3.6.
- nustr = additional_substitutions(nustr, is_search);
-
- //transformations based on Unicode category codes
- nustr = transformations_on_unicode(nustr, usf);
-
- //since we've stripped out the control characters, we can now
- //use a few as placeholders temporarily
- nustr = replace_placehoders(nustr);
-
- //decimal digit
- nustr = decimal_digits(nustr);
-
- //intentionally skipping step 8 of the NACO algorithm; if the string
- //gets normalized away, that's fine.
-
- //leading and trailing spaces
- nustr = leading_trailing_spaces(nustr);
- nustr_len = u_strlen(nustr);
-
- temp = palloc(nustr_len * sizeof(UChar));
- u_strToLower(temp, nustr_len, nustr, nustr_len, NULL, &err);
- temp_len = nustr_len;
- err = 0;
-
- u_strToUTF8(NULL, 0, &str_len, temp, temp_len, &err);
-
- err = 0;
- str = (text *)palloc(str_len + VARHDRSZ);
- SET_VARSIZE(str, str_len + VARHDRSZ);
-
- result = u_strToUTF8(VARDATA(str), str_len, &str_len, temp, temp_len, &err);
-
- if(result != NULL)
- result[str_len] = '\0';
-
-
-Fail:
- if(temp != NULL)
- pfree(temp);
- if(ustr != NULL)
- pfree(ustr);
- if(usf != NULL)
- pfree(usf);
- if(nustr != NULL)
- pfree(nustr);
-
- return str;
-}
-
-PG_FUNCTION_INFO_V1(naco_normalize);
-
-Datum naco_normalize(PG_FUNCTION_ARGS)
-{
- text *str = PG_GETARG_TEXT_P(0);
- text *sf = PG_GETARG_TEXT_P(1);
-
-
- if (str == NULL || sf == NULL)
- {
- PG_RETURN_TEXT_P(NULL);
- }
-
- str = normalize(str, sf, 0);
-
- if(str != NULL)
- PG_RETURN_TEXT_P(str);
- else
- PG_RETURN_NULL();
-}
-
-PG_FUNCTION_INFO_V1(search_normalize);
-
-Datum search_normalize(PG_FUNCTION_ARGS)
-{
- text *str = PG_GETARG_TEXT_P(0);
- text *sf = PG_GETARG_TEXT_P(1);
-
-
- if (str == NULL || sf == NULL)
- {
- PG_RETURN_TEXT_P(NULL);
- }
-
- str = normalize(str, sf, 1);
-
-
- if(str != NULL)
- PG_RETURN_TEXT_P(str);
- else
- PG_RETURN_NULL();
-}
+/************************************************************************/\r
+/* C Implementation: public.search_normalize public.naco_normalize\r
+ *\r
+ * Descritption:\r
+ * This file implement public.search_normalize and public.naco_normalize.\r
+ * These two functions is included in the PostgreSQL extension c_functions.\r
+ * ICU4C and postgres lib is needed to build this file.\r
+ *\r
+ * Author: Swenyu Duan <dsy88@sina.com>, (C) 2012\r
+ *\r
+ * Copyright: See COPYING file that comes with this distribution.\r
+ */\r
+/************************************************************************/\r
+#include "postgres.h"\r
+#include "string.h"\r
+#include "fmgr.h"\r
+#include "unicode/unorm2.h"\r
+#include "unicode/utypes.h"\r
+#include "unicode/ustring.h"\r
+#include "unicode/uregex.h"\r
+#include "unicode/umachine.h"\r
+\r
+#ifdef PG_MODULE_MAGIC\r
+PG_MODULE_MAGIC;\r
+#endif\r
+\r
+static int32_t regexp_transliterate(const UChar *search_list,\r
+ int32_t search_list_len,\r
+ const UChar *replacement_list,\r
+ int32_t replacement_list_len,\r
+ UChar *src,\r
+ int32_t src_len,\r
+ UChar *des,\r
+ int32_t des_capacity)\r
+{\r
+ int i, j;\r
+ int32_t des_len;\r
+ UChar *cur_pos;\r
+\r
+ if (search_list == NULL || \r
+ replacement_list == NULL ||\r
+ des == NULL ||\r
+ src_len > des_capacity + 1)\r
+ {\r
+ return 0;\r
+ }\r
+\r
+ des_len = 0;\r
+\r
+ for (i = 0; i < src_len; i++)\r
+ {\r
+ if (des != NULL)\r
+ {\r
+ des[des_len] = src[i];\r
+ }\r
+ des_len++;\r
+\r
+ for (j = replacement_list_len; j < search_list_len; j++)\r
+ {\r
+ if (search_list[j] == src[i])\r
+ {\r
+ des_len--;\r
+ break;\r
+ }\r
+ }\r
+ }\r
+ if (des == NULL)\r
+ {\r
+ //To store the tail '\\0'.\r
+ return des_len + 1;\r
+ }\r
+\r
+ des[des_len] = '\0';\r
+ \r
+ for (i = 0; i < replacement_list_len; i++)\r
+ {\r
+ cur_pos = u_strchr(des, search_list[i]);\r
+\r
+ while (cur_pos != NULL)\r
+ {\r
+ *cur_pos = replacement_list[i];\r
+\r
+ //In case cur_pos is the last char in des.\r
+ if (cur_pos >= des + des_len)\r
+ {\r
+ break;\r
+ }\r
+ cur_pos = u_strchr(cur_pos + 1, search_list[i]);\r
+ }\r
+ }\r
+\r
+ return des_len;\r
+}\r
+\r
+static int32_t regexp_replace(const UChar *regexp,\r
+ int32_t regexp_len,\r
+ const UChar *replacement,\r
+ int32_t replacement_len,\r
+ UChar *src,\r
+ int32_t src_len,\r
+ UChar *des,\r
+ int32_t des_capacity,\r
+ int is_global)\r
+{\r
+ URegularExpression *regular_exp;\r
+ UErrorCode status;\r
+ UParseError pe;\r
+ int32_t len;\r
+\r
+ if (regexp == NULL || replacement == NULL || src == NULL)\r
+ {\r
+ return 0;\r
+ }\r
+\r
+ regular_exp = uregex_open(regexp, regexp_len, 0, &pe, &status);\r
+ if (regular_exp == NULL)\r
+ {\r
+ return 0;\r
+ }\r
+\r
+ uregex_setText(regular_exp, src, src_len, &status);\r
+\r
+ if (is_global > 0)\r
+ {\r
+ len = uregex_replaceAll(regular_exp,\r
+ replacement,\r
+ replacement_len,\r
+ NULL,\r
+ 0,\r
+ &status);\r
+ if (des == NULL || des_capacity < len)\r
+ {\r
+ uregex_close(regular_exp);\r
+ return len;\r
+ }\r
+\r
+ uregex_replaceAll(regular_exp,\r
+ replacement,\r
+ replacement_len,\r
+ des,\r
+ des_capacity,\r
+ &status);\r
+ }\r
+ else\r
+ {\r
+ len = uregex_replaceFirst(regular_exp,\r
+ replacement,\r
+ replacement_len,\r
+ NULL,\r
+ 0,\r
+ &status);\r
+ if (des == NULL || des_capacity < len)\r
+ {\r
+ uregex_close(regular_exp);\r
+ return len;\r
+ }\r
+\r
+ uregex_replaceFirst(regular_exp,\r
+ replacement,\r
+ replacement_len,\r
+ des,\r
+ des_capacity,\r
+ &status);\r
+ }\r
+ \r
+ uregex_close(regular_exp);\r
+ return len;\r
+}\r
+\r
+static UChar *u_strtransliterate(UChar *search_list,\r
+ UChar *replacement_list,\r
+ UChar *str,\r
+ int32_t str_capacity)\r
+{\r
+ int32_t search_list_len, replacement_list_len, str_len;\r
+ UChar *des;\r
+ int32_t des_len;\r
+\r
+ if (search_list == NULL || replacement_list == NULL || str == NULL)\r
+ {\r
+ return NULL;\r
+ }\r
+\r
+ search_list_len = u_strlen(search_list);\r
+ replacement_list_len = u_strlen(replacement_list);\r
+ str_len = u_strlen(str);\r
+\r
+ des_len = regexp_transliterate(search_list, \r
+ search_list_len,\r
+ replacement_list,\r
+ replacement_list_len,\r
+ str,\r
+ str_len,\r
+ NULL,\r
+ 0);\r
+ des = palloc(des_len * sizeof(UChar));\r
+ des_len = regexp_transliterate(search_list, \r
+ search_list_len,\r
+ replacement_list,\r
+ replacement_list_len,\r
+ str,\r
+ str_len,\r
+ des,\r
+ des_len);\r
+\r
+ pfree(str);\r
+ return des;\r
+}\r
+\r
+static UChar *u_strreplace(UChar *regexp,\r
+ UChar *replacement,\r
+ UChar *str,\r
+ int32_t str_capacity,\r
+ int is_global)\r
+{\r
+ int32_t regexp_len, replacement_len, str_len;\r
+ UChar *des;\r
+ int32_t des_len;\r
+\r
+ if (regexp == NULL || replacement == NULL || str == NULL)\r
+ {\r
+ return NULL;\r
+ }\r
+\r
+ regexp_len = u_strlen(regexp);\r
+ replacement_len = u_strlen(replacement);\r
+ str_len = u_strlen(str);\r
+\r
+ des_len = regexp_replace(regexp,\r
+ regexp_len,\r
+ replacement,\r
+ replacement_len,\r
+ str,\r
+ str_len,\r
+ NULL,\r
+ 0,\r
+ is_global);\r
+\r
+ des = palloc(des_len * sizeof(UChar));\r
+\r
+ des_len = regexp_replace(regexp,\r
+ regexp_len,\r
+ replacement,\r
+ replacement_len,\r
+ str,\r
+ str_len,\r
+ des,\r
+ des_len,\r
+ is_global);\r
+ pfree(str);\r
+ return des;\r
+}\r
+\r
+UChar *additional_substitutions(UChar *nustr, int is_search)\r
+{\r
+ char *regexp;\r
+ UChar uregexp[200], replacement[200];\r
+\r
+ if (nustr == NULL)\r
+ {\r
+ return NULL;\r
+ }\r
+\r
+ regexp = "\\x{00C6}";\r
+ u_uastrncpy(uregexp, regexp, strlen(regexp));\r
+ u_uastrncpy(replacement, "AE", strlen("AE"));\r
+ \r
+ nustr = u_strreplace(uregexp, replacement, nustr, u_strlen(nustr), 1);\r
+\r
+ regexp = "\\x{00DE}";\r
+ u_uastrncpy(uregexp, regexp, strlen(regexp));\r
+ u_uastrncpy(replacement, "TH", strlen("TH"));\r
+ \r
+ nustr = u_strreplace(uregexp, replacement, nustr, u_strlen(nustr), 1);\r
+\r
+ regexp = "\\x{0152}";\r
+ u_uastrncpy(uregexp, regexp, strlen(regexp));\r
+ u_uastrncpy(replacement, "OE", strlen("OE"));\r
+\r
+ nustr = u_strreplace(uregexp, replacement, nustr, u_strlen(nustr), 1);\r
+\r
+ if (is_search)\r
+ {\r
+ regexp = "\\x{0110}\\x{00D0}\\x{00D8}\\x{0141}\\x{2113}\\x{02BB}\\x{02BC}][";\r
+ }\r
+ else\r
+ {\r
+ regexp = "\\x{0110}\\x{00D0}\\x{00D8}\\x{0141}\\x{2113}\\x{02BB}\\x{02BC}]['";\r
+ }\r
+ \r
+ u_uastrncpy(uregexp, regexp, strlen(regexp));\r
+ u_uastrncpy(replacement, "OE", strlen("OE"));\r
+ \r
+ nustr = u_strreplace(uregexp, replacement, nustr, u_strlen(nustr), 1);\r
+\r
+ regexp = "\\x{0110}\\x{00D0}\\x{00D8}\\x{0141}\\x{2113}\\x{02BB}\\x{02BC}]['";\r
+ u_uastrncpy(uregexp, regexp, strlen(regexp));\r
+ u_uastrncpy(replacement, "DDOLl", strlen("DDOLl"));\r
+ \r
+ nustr = u_strtransliterate(uregexp, replacement, nustr, u_strlen(nustr));\r
+\r
+ return nustr;\r
+}\r
+\r
+UChar *transformations_on_unicode(UChar *nustr, UChar *usf)\r
+{\r
+ char *regexp;\r
+ UChar uregexp[200], replacement[200];\r
+ UChar *comma;\r
+ int32_t nustr_len;\r
+\r
+ if (nustr == NULL)\r
+ {\r
+ return NULL;\r
+ }\r
+\r
+ nustr_len = u_strlen(nustr);\r
+\r
+ regexp = "[\\p{Cc}\\p{Cf}\\p{Co}\\p{Cs}\\p{Lm}\\p{Mc}\\p{Me}\\p{Mn}]";\r
+ u_uastrncpy(uregexp, regexp, strlen(regexp));\r
+ u_uastrncpy(replacement, "", strlen(""));\r
+ \r
+ nustr = u_strreplace(uregexp, replacement, nustr, u_strlen(nustr), 1);\r
+\r
+ if (usf != NULL && usf[0] == 0x61) //0x61 == 'a' in utf16\r
+ {\r
+ comma = u_strchr(nustr, 0x2c); //0x2c == ',' in utf16\r
+ if (comma != NULL)\r
+ {\r
+ if (comma != nustr + nustr_len - 1)\r
+ {\r
+ regexp = ",";\r
+ u_uastrncpy(uregexp, regexp, strlen(regexp));\r
+ replacement[0] = 0x7;\r
+ replacement[1] = 0;\r
+\r
+ nustr = u_strreplace(uregexp, replacement, nustr, u_strlen(nustr), 0);\r
+ }\r
+ }\r
+ }\r
+\r
+ return nustr;\r
+}\r
+\r
+UChar *replace_placehoders(UChar *nustr)\r
+{\r
+ UChar uregexp[200], replacement[200];\r
+\r
+ if (nustr == NULL)\r
+ {\r
+ return NULL;\r
+ }\r
+\r
+ u_uastrncpy(uregexp,\r
+ "+&@\\x{266D}\\x{266F}#",\r
+ strlen( "+&@\\x{266D}\\x{266F}#"));\r
+ u_uastrncpy(replacement,\r
+ "\\x01\\x02\\x03\\x04\\x05\\x06",\r
+ strlen("\\x01\\x02\\x03\\x04\\x05\\x06"));\r
+\r
+ nustr = u_strtransliterate(uregexp, replacement, nustr, u_strlen(nustr));\r
+\r
+ u_uastrncpy(uregexp,\r
+ "[\\p{Pc}\\p{Pd}\\p{Pe}\\p{Pf}\\p{Pi}\\p{Po}\\p{Ps}\\p{Sk}\\p{Sm}\\p{So}\\p{Zl}\\p{Zp}\\p{Zs}]",\r
+ strlen( "[\\p{Pc}\\p{Pd}\\p{Pe}\\p{Pf}\\\\p{Pi}\\p{Po}\\p{Ps}\\p{Sk}\\p{Sm}\\p{So}\\p{Zl}\\p{Zp}\\p{Zs}]"));\r
+ u_uastrncpy(replacement, "", strlen(""));\r
+\r
+ nustr = u_strreplace(uregexp, replacement, nustr, u_strlen(nustr), 1);\r
+\r
+ u_uastrncpy(uregexp,\r
+ "\\x01\\x02\\x03\\x04\\x05\\x06\\x07",\r
+ strlen( "\\x01\\x02\\x03\\x04\\x05\\x06\\x07"));\r
+ u_uastrncpy(replacement,\r
+ "+&@\\x{266D}\\x{266F}#,",\r
+ strlen("+&@\\x{266D}\\x{266F}#,"));\r
+\r
+ nustr = u_strtransliterate(uregexp, replacement, nustr, u_strlen(nustr));\r
+\r
+\r
+ return nustr;\r
+}\r
+\r
+UChar *decimal_digits(UChar *nustr)\r
+{\r
+ UChar uregexp[300], replacement[300];\r
+\r
+ if (nustr == NULL)\r
+ {\r
+ return NULL;\r
+ }\r
+ \r
+ u_uastrncpy(uregexp,\r
+ "\\x{0660}-\\x{0669}\\x{06F0}-\\x{06F9}\\x{07C0}-\\x{07C9}\\x{0966}-\\x{096F}"\r
+ "\\x{09E6}-\\x{09EF}\\x{0A66}-\\x{0A6F}\\x{0AE6}-\\x{0AEF}\\x{0B66}-\\x{0B6F}"\r
+ "\\x{0BE6}-\\x{0BEF}\\x{0C66}-\\x{0C6F}\\x{0CE6}-\\x{0CEF}\\x{0D66}-\\x{0D6F}"\r
+ "\\x{0E50}-\\x{0E59}\\x{0ED0}-\\x{0ED9}\\x{0F20}-\\x{0F29}\\x{1040}-\\x{1049}"\r
+ "\\x{1090}-\\x{1099}\\x{17E0}-\\x{17E9}\\x{1810}-\\x{1819}\\x{1946}-\\x{194F}"\r
+ "\\x{19D0}-\\x{19D9}\\x{1A80}-\\x{1A89}\\x{1A90}-\\x{1A99}\\x{1B50}-\\x{1B59}"\r
+ "\\x{1BB0}-\\x{1BB9}\\x{1C40}-\\x{1C49}\\x{1C50}-\\x{1C59}\\x{A620}-\\x{A629}"\r
+ "\\x{A8D0}-\\x{A8D9}\\x{A900}-\\x{A909}\\x{A9D0}-\\x{A9D9}\\x{AA50}-\\x{AA59}"\r
+ "\\x{ABF0}-\\x{ABF9}\\x{FF10}-\\x{FF19}",\r
+ strlen("\\x{0660}-\\x{0669}\\x{06F0}-\\x{06F9}\\x{07C0}-\\x{07C9}\\x{0966}-\\x{096F}"\r
+ "\\x{09E6}-\\x{09EF}\\x{0A66}-\\x{0A6F}\\x{0AE6}-\\x{0AEF}\\x{0B66}-\\x{0B6F}"\r
+ "\\x{0BE6}-\\x{0BEF}\\x{0C66}-\\x{0C6F}\\x{0CE6}-\\x{0CEF}\\x{0D66}-\\x{0D6F}"\r
+ "\\x{0E50}-\\x{0E59}\\x{0ED0}-\\x{0ED9}\\x{0F20}-\\x{0F29}\\x{1040}-\\x{1049}"\r
+ "\\x{1090}-\\x{1099}\\x{17E0}-\\x{17E9}\\x{1810}-\\x{1819}\\x{1946}-\\x{194F}"\r
+ "\\x{19D0}-\\x{19D9}\\x{1A80}-\\x{1A89}\\x{1A90}-\\x{1A99}\\x{1B50}-\\x{1B59}"\r
+ "\\x{1BB0}-\\x{1BB9}\\x{1C40}-\\x{1C49}\\x{1C50}-\\x{1C59}\\x{A620}-\\x{A629}"\r
+ "\\x{A8D0}-\\x{A8D9}\\x{A900}-\\x{A909}\\x{A9D0}-\\x{A9D9}\\x{AA50}-\\x{AA59}"\r
+ "\\x{ABF0}-\\x{ABF9}\\x{FF10}-\\x{FF19}"));\r
+\r
+ u_uastrncpy(replacement,\r
+ "0-90-90-90-90-90-90-90-90-90-90-90-90-90-90-90-90-90-90-90-90-90-90-90-90-90-90-90-90-90-90-90-90-90-9",\r
+ strlen("0-90-90-90-90-90-90-90-90-90-90-90-90-90-90-90-90-90-90-90-90-90-90-90-90-90-90-90-90-90-90-90-90-90-9"));\r
+\r
+ nustr = u_strtransliterate(uregexp, replacement, nustr, u_strlen(nustr));\r
+\r
+ return nustr;\r
+}\r
+\r
+UChar *leading_trailing_spaces(UChar * nustr)\r
+{\r
+ UChar uregexp[200], replacement[200];\r
+\r
+ if (nustr == NULL)\r
+ {\r
+ return NULL;\r
+ }\r
+\r
+ u_uastrncpy(uregexp, "\\s+", strlen( "\\s+"));\r
+ u_uastrncpy(replacement, " ", strlen(" "));\r
+\r
+ nustr = u_strreplace(uregexp, replacement, nustr, u_strlen(nustr), 1);\r
+ \r
+ u_uastrncpy(uregexp, "^\\s+", strlen( "^\\s+"));\r
+ u_uastrncpy(replacement, "", strlen(""));\r
+ nustr = u_strreplace(uregexp, replacement, nustr, u_strlen(nustr), 0);\r
+\r
+ u_uastrncpy(uregexp, "\\s+$", strlen( "\\s+$"));\r
+\r
+ nustr = u_strreplace(uregexp, replacement, nustr, u_strlen(nustr), 1);\r
+\r
+ return nustr;\r
+\r
+}\r
+\r
+text *normalize(text *str, text *sf, int is_search)\r
+{\r
+ UChar *ustr, *nustr, *temp, *usf;\r
+ int32_t nustr_len, temp_len, str_len;\r
+ UNormalizer2 *normalizer;\r
+ char *regexp, *result;\r
+ UChar uregexp[200], replacement[200];\r
+ UErrorCode err;\r
+\r
+ if (str == NULL || sf == NULL)\r
+ {\r
+ return NULL;\r
+ }\r
+\r
+ normalizer = (UNormalizer2 *)unorm2_getNFKDInstance(&err);\r
+ if (U_FAILURE(err))\r
+ {\r
+ return NULL;\r
+ }\r
+\r
+ ustr = palloc(VARSIZE(str) * sizeof(UChar));\r
+ nustr = palloc(VARSIZE(str)* sizeof(UChar));\r
+ temp = palloc(VARSIZE(str) * sizeof(UChar));\r
+ usf = palloc(VARSIZE(sf) * sizeof(UChar));\r
+\r
+ temp = u_strFromUTF8(temp, VARSIZE(str), NULL, VARDATA(str), VARSIZE(str), &err);\r
+ if (U_FAILURE(err) || temp == NULL)\r
+ {\r
+ str = NULL;\r
+ goto Fail;\r
+ }\r
+\r
+ nustr = u_strncpy(nustr, temp, VARSIZE(str));\r
+ if (nustr == NULL)\r
+ {\r
+ str = NULL;\r
+ goto Fail;\r
+ }\r
+\r
+ usf = u_strFromUTF8(usf, VARSIZE(sf), NULL, VARDATA(sf), VARSIZE(sf), &err);\r
+ if (usf == NULL)\r
+ {\r
+ str = NULL;\r
+ goto Fail;\r
+ }\r
+ //Apply NACO normalization to input string; based on\r
+ //http://www.loc.gov/catdir/pcc/naco/SCA_PccNormalization_Final_revised.pdf\r
+ //\r
+ //Note that unlike a strict reading of the NACO normalization rules,\r
+ //output is returned as lowercase instead of uppercase for compatibility\r
+ //with previous versions of the Evergreen naco_normalize routine.\r
+ //\r
+ //Convert to upper-case first; even though final output will be lowercase, doing this will\r
+ //ensure that the German eszett (?) and certain ligatures (?, ?, ?, etc.) will be handled correctly.\r
+ //If there are any bugs in Perl's implementation of upcasing, they will be passed through here.\r
+\r
+ u_strToUpper(ustr, \r
+ VARSIZE(str),\r
+ temp,\r
+ VARSIZE(str),\r
+ NULL,\r
+ &err);\r
+ pfree(temp);\r
+ temp = NULL;\r
+ if (U_FAILURE(err))\r
+ {\r
+ str = NULL;\r
+ goto Fail;\r
+ }\r
+\r
+ regexp = "\\x{0098}.*?\\x{009C}";\r
+ u_uastrncpy(uregexp, regexp, strlen(regexp));\r
+ u_uastrncpy(replacement, "", strlen(""));\r
+\r
+ ustr = u_strreplace(uregexp, replacement, ustr, u_strlen(ustr), 1);\r
+\r
+ unorm2_normalize(normalizer, ustr, VARSIZE(str), nustr, VARSIZE(str), &err);\r
+ if (U_FAILURE(err))\r
+ {\r
+ str = NULL;\r
+ goto Fail;\r
+ }\r
+\r
+ //additional substitutions - 3.6.\r
+ nustr = additional_substitutions(nustr, is_search);\r
+\r
+ //transformations based on Unicode category codes\r
+ nustr = transformations_on_unicode(nustr, usf);\r
+\r
+ //since we've stripped out the control characters, we can now\r
+ //use a few as placeholders temporarily\r
+ nustr = replace_placehoders(nustr);\r
+\r
+ //decimal digit\r
+ nustr = decimal_digits(nustr);\r
+\r
+ //intentionally skipping step 8 of the NACO algorithm; if the string\r
+ //gets normalized away, that's fine.\r
+\r
+ //leading and trailing spaces\r
+ nustr = leading_trailing_spaces(nustr);\r
+ nustr_len = u_strlen(nustr);\r
+\r
+ temp = palloc(nustr_len * sizeof(UChar));\r
+ u_strToLower(temp, nustr_len, nustr, nustr_len, NULL, &err);\r
+ temp_len = nustr_len;\r
+\r
+\r
+ u_strToUTF8(NULL, 0, &str_len, temp, temp_len, &err);\r
+\r
+ SET_VARSIZE(str, str_len + VARHDRSZ);\r
+ str = (text *)palloc(str_len + VARHDRSZ);\r
+ \r
+ result = u_strToUTF8(VARDATA(str), str_len, &str_len, temp, temp_len, &err); \r
+\r
+ result[str_len] = '\0';\r
+\r
+ \r
+Fail:\r
+ pfree(temp);\r
+ pfree(ustr);\r
+ pfree(nustr);\r
+\r
+ return str;\r
+}\r
+\r
+PG_FUNCTION_INFO_V1(naco_normalize);\r
+\r
+Datum naco_normalize(PG_FUNCTION_ARGS)\r
+{\r
+ text *str = PG_GETARG_TEXT_P(0);\r
+ text *sf = PG_GETARG_TEXT_P(1);\r
+ \r
+\r
+ if (str == NULL || sf == NULL)\r
+ {\r
+ PG_RETURN_TEXT_P(NULL);\r
+ }\r
+\r
+ str = normalize(str, sf, 0);\r
+ \r
+ PG_RETURN_TEXT_P(str);\r
+}\r
+\r
+PG_FUNCTION_INFO_V1(search_normalize);\r
+\r
+Datum search_normalize(PG_FUNCTION_ARGS)\r
+{\r
+ text *str = PG_GETARG_TEXT_P(0);\r
+ text *sf = PG_GETARG_TEXT_P(1);\r
+\r
+\r
+ if (str == NULL || sf == NULL)\r
+ {\r
+ PG_RETURN_TEXT_P(NULL);\r
+ }\r
+\r
+ str = normalize(str, sf, 1);\r
+\r
+ PG_RETURN_TEXT_P(str);\r
+}\r
+\r
projects / working / Evergreen.git / commitdiff