refactoring, to avoid creating/deleting zillions of UnicodeNormalizers

kosloot · kosloot · commit c5c8851f1cd9 · 2026-02-26T09:01:55.000+01:00
diff --git a/configure.ac b/configure.ac
@@ -108,7 +108,7 @@ LIBS="$ICU_LIBS $LIBS"
 
 AX_LIB_READLINE
 
-PKG_CHECK_MODULES([ticcutils], [ticcutils >= 0.36] )
+PKG_CHECK_MODULES([ticcutils], [ticcutils >= 0.38] )
 
 CXXFLAGS="$CXXFLAGS $ticcutils_CFLAGS"
 LIBS="$ticcutils_LIBS $LIBS"
diff --git a/include/frog/Parser.h b/include/frog/Parser.h
@@ -82,6 +82,7 @@ class ParserBase {
   std::string MWU_tagset;
   std::string textclass;
   TiCC::UniFilter *filter;
+  TiCC::UnicodeNormalizer _normalizer;
   std::string _host;
   std::string _port;
 };
@@ -107,6 +108,8 @@ class Parser: public ParserBase {
   std::vector<icu::UnicodeString> createRelInstances( const parseData& );
   std::vector<timbl_result> timbl_server( const std::string&,
 					  const std::vector<icu::UnicodeString>& );
+  std::vector<timbl_result> timbl( Timbl::TimblAPI *,
+				   const std::vector<UnicodeString>& );
   Parser( const Parser& ) = delete; // inhibit copies
   Parser operator=( const Parser& ) = delete; // inhibit copies
   std::string maxDepSpanS;
diff --git a/include/frog/clex.h b/include/frog/clex.h
@@ -62,7 +62,7 @@ namespace CLEX {
   };
   bool is_CELEX_base( const Type& );
   Type select_tag( const char ch );
-  std::string toString( const Type& );
+  icu::UnicodeString toString( const Type& );
   icu::UnicodeString toUnicodeString( const Type& );
   Type toCLEX( const icu::UnicodeString& );
   Type toCLEX( const UChar );
diff --git a/include/frog/mblem_mod.h b/include/frog/mblem_mod.h
@@ -94,6 +94,7 @@ class Mblem {
   TiCC::LogStream *errLog;
   TiCC::LogStream *dbgLog;
   TiCC::UniFilter *filter;
+  mutable TiCC::UnicodeNormalizer _normalizer;
   Mblem( const Mblem& ) = delete;
   Mblem& operator=( const Mblem& ) = delete;
 };
diff --git a/include/frog/mbma_mod.h b/include/frog/mbma_mod.h
@@ -114,6 +114,7 @@ class Mbma {
   TiCC::LogStream *errLog;
   TiCC::LogStream *dbgLog;
   TiCC::UniFilter *filter;
+  mutable TiCC::UnicodeNormalizer _normalizer;
   std::string _host;
   std::string _port;
   std::string _base;
diff --git a/include/frog/ner_tagger_mod.h b/include/frog/ner_tagger_mod.h
@@ -81,6 +81,7 @@ class NERTagger: public BaseTagger {
 		  std::vector<std::map<icu::UnicodeString,std::set<std::string>>>& );
   std::vector<icu::UnicodeString> create_ner_list( const std::vector<icu::UnicodeString>&,
 						   const std::vector<std::map<icu::UnicodeString,std::set<std::string>>>& );
+  std::vector<UnicodeString> serialize( const std::vector<std::set<std::string>>& ) const;
   std::vector<std::map<icu::UnicodeString,std::set<std::string>>> gazet_ners;
   std::vector<std::map<icu::UnicodeString,std::set<std::string>>> override_ners;
   void addEntity( frog_data&,
diff --git a/include/frog/tagger_base.h b/include/frog/tagger_base.h
@@ -70,6 +70,8 @@ class BaseTagger {
   const std::string& version() const { return _version; };
  private:
   std::vector<tag_entry> extract_sentence( const frog_data& );
+  nlohmann::json create_json( const std::vector<tag_entry>& ) const;
+  std::vector<Tagger::TagResult> json_to_TR( const nlohmann::json& in ) const;
   nlohmann::json read_from_client( Sockets::ClientSocket& ) const;
   void write_to_client( nlohmann::json&,
 			Sockets::ClientSocket& ) const;
@@ -87,6 +89,7 @@ class BaseTagger {
   std::string _port;
   MbtAPI *tagger;
   TiCC::UniFilter *filter;
+  mutable TiCC::UnicodeNormalizer _normalizer;
   std::vector<std::string> _words;
   std::vector<Tagger::TagResult> _tag_result;
   std::map<icu::UnicodeString,icu::UnicodeString> token_tag_map;
diff --git a/src/FrogData.cxx b/src/FrogData.cxx
@@ -68,10 +68,11 @@ json frog_record::to_json() const {
     \return an JSON structure
   */
   json result;
-  result["word"] = TiCC::UnicodeToUTF8(word);
+  TiCC::UnicodeNormalizer UN;
+  result["word"] = TiCC::UnicodeToUTF8(word,UN);
   if ( !token_class.isEmpty() ){
     json tok;
-    tok["token"] = TiCC::UnicodeToUTF8(token_class);
+    tok["token"] = TiCC::UnicodeToUTF8(token_class,UN);
     if ( no_space ){
       tok["space"] = false;
     }
@@ -81,29 +82,29 @@ json frog_record::to_json() const {
     result["ucto"] = tok;
   }
   if ( !lemmas.empty() ){
-    result["lemma"] = TiCC::UnicodeToUTF8(lemmas[0]);
+    result["lemma"] = TiCC::UnicodeToUTF8(lemmas[0],UN);
   }
   if ( !morph_string.isEmpty() ){
-    result["morph"] = TiCC::UnicodeToUTF8(morph_string);
+    result["morph"] = TiCC::UnicodeToUTF8(morph_string,UN);
   }
   if ( compound_string.find("0") == string::npos ){
     result["compound"] = compound_string;
   }
   if ( !tag.isEmpty() ){
     json tg;
-    tg["tag"] = TiCC::UnicodeToUTF8(tag);
+    tg["tag"] = TiCC::UnicodeToUTF8(tag,UN);
     tg["confidence"] = tag_confidence;
     result["pos"] = tg;
   }
   if ( !ner_tag.isEmpty() && ner_confidence > 0.0 ){
     json tg;
-    tg["tag"] = TiCC::UnicodeToUTF8(ner_tag);
+    tg["tag"] = TiCC::UnicodeToUTF8(ner_tag,UN);
     tg["confidence"] = ner_confidence;
     result["ner"] = tg;
   }
   if ( !iob_tag.isEmpty() ){
     json tg;
-    tg["tag"] = TiCC::UnicodeToUTF8(iob_tag);
+    tg["tag"] = TiCC::UnicodeToUTF8(iob_tag,UN);
     tg["confidence"] = iob_confidence;
     result["chunking"] = tg;
   }
diff --git a/src/Parser.cxx b/src/Parser.cxx
@@ -984,8 +984,8 @@ parseData Parser::prepareParse( frog_data& fd ){         //     |
 }
 
 
-vector<timbl_result> timbl( Timbl::TimblAPI* tim,
-			    const vector<UnicodeString>& instances ){
+vector<timbl_result> Parser::timbl( Timbl::TimblAPI* tim,
+				    const vector<UnicodeString>& instances ){
   /// call a Timbl experiment with a list of instances
   /*!
     \param tim The Timbl to use
@@ -997,7 +997,7 @@ vector<timbl_result> timbl( Timbl::TimblAPI* tim,
   for ( const auto& inst : instances ){
     const Timbl::ClassDistribution *db;
     const Timbl::TargetValue *tv = tim->Classify( inst, db );
-    result.push_back( timbl_result( TiCC::UnicodeToUTF8(tv->name()),
+    result.push_back( timbl_result( TiCC::UnicodeToUTF8(tv->name(),_normalizer),
 				    db->Confidence(tv), *db ) );
   }
   return result;
@@ -1073,7 +1073,7 @@ vector<timbl_result> Parser::timbl_server( const string& base,
   query["command"] = "classify";
   json arr = json::array();
   for ( const auto& inst : instances ){
-    arr.push_back( TiCC::UnicodeToUTF8(inst) );
+    arr.push_back( TiCC::UnicodeToUTF8(inst,_normalizer) );
   }
   query["params"] = arr;
   DBG << "send json" << query.dump(2) << endl;
diff --git a/src/cgn_tagger_mod.cxx b/src/cgn_tagger_mod.cxx
@@ -300,7 +300,7 @@ void CGNTagger::add_tags( const vector<folia::Word*>& wv,
   for ( const auto& word : fd.units ){
     folia::KWargs u_args;
     u_args["set"]   = getTagset();
-    u_args["class"] = TiCC::UnicodeToUTF8(word.tag);
+    u_args["class"] = TiCC::UnicodeToUTF8(word.tag,_normalizer);
     if ( textclass != "current" ){
       u_args["textclass"] = textclass;
     }
@@ -312,7 +312,7 @@ void CGNTagger::add_tags( const vector<folia::Word*>& wv,
     }
     vector<UnicodeString> hv = TiCC::split_at_first_of( word.tag, "()" );
     UnicodeString head = hv[0];
-    u_args["class"] = TiCC::UnicodeToUTF8(head);
+    u_args["class"] = TiCC::UnicodeToUTF8(head,_normalizer);
 #pragma omp critical (foliaupdate)
     {
       postag->add_child<folia::HeadFeature>( u_args );
@@ -325,10 +325,10 @@ void CGNTagger::add_tags( const vector<folia::Word*>& wv,
       for ( const auto& f : feats ){
 	folia::KWargs f_args;
 	f_args["set"] =  getTagset();
-	f_args["subset"] = getSubSet( TiCC::UnicodeToUTF8(f),
-				      TiCC::UnicodeToUTF8(head),
-				      TiCC::UnicodeToUTF8(word.tag) );
-	f_args["class"]  = TiCC::UnicodeToUTF8(f);
+	f_args["subset"] = getSubSet( TiCC::UnicodeToUTF8(f,_normalizer),
+				      TiCC::UnicodeToUTF8(head,_normalizer),
+				      TiCC::UnicodeToUTF8(word.tag,_normalizer) );
+	f_args["class"]  = TiCC::UnicodeToUTF8(f,_normalizer);
 #pragma omp critical (foliaupdate)
 	{
 	  postag->add_child<folia::Feature>( f_args );
diff --git a/src/clex.cxx b/src/clex.cxx
@@ -130,8 +130,8 @@ namespace CLEX {
     return toCLEX(s);
   }
 
-  string toString( const Type& t ){
-    /// convert a CLEX::Type to a string
+  UnicodeString toString( const Type& t ){
+    /// convert a CLEX::Type to a UnicodeString
     /*!
       \param t a CLEX::Type value
       \return a string representing the type, in a format that can be converted
@@ -163,7 +163,7 @@ namespace CLEX {
   }
 
   UnicodeString toUnicodeString( const Type& t ){
-    return TiCC::UnicodeFromUTF8( toString( t ) );
+    return toString( t );
   }
 
   bool is_CELEX_base( const Type& t ){
diff --git a/src/iob_tagger_mod.cxx b/src/iob_tagger_mod.cxx
@@ -215,7 +215,7 @@ void IOBTagger::add_result( const frog_data& fd,
       if ( !el->id().empty() ){
 	args["generate_id"] = el->id();
       }
-      args["class"] = TiCC::UnicodeToUTF8(word.iob_tag).substr(2);
+      args["class"] = TiCC::UnicodeToUTF8(word.iob_tag,_normalizer).substr(2);
       args["confidence"] = TiCC::toString(word.iob_confidence);
       if ( textclass != "current" ){
 	args["textclass"] = textclass;
diff --git a/src/mblem_mod.cxx b/src/mblem_mod.cxx
@@ -206,7 +206,8 @@ bool Mblem::init( const TiCC::Configuration& config ) {
     }
   }
 
-  UnicodeString one_one_tagS = TiCC::UnicodeFromUTF8(config.lookUp( "one_one_tags", "mblem" ));
+  UnicodeString one_one_tagS
+    = TiCC::UnicodeFromUTF8(config.lookUp( "one_one_tags", "mblem" ), _normalizer );
   if ( !one_one_tagS.isEmpty() ){
     vector<UnicodeString> tags = TiCC::split_at( one_one_tagS, "," );
     for ( auto const& t : tags ){
@@ -527,7 +528,7 @@ UnicodeString Mblem::call_server( const UnicodeString& instance ){
   // create json query struct
   json query;
   query["command"] = "classify";
-  query["param"] = TiCC::UnicodeToUTF8(instance);
+  query["param"] = TiCC::UnicodeToUTF8(instance,_normalizer);
   //  LOG << "send json" << query.dump(2) << endl;
   // send it to the server
   string out_line = query.dump() + "\n";
@@ -546,7 +547,7 @@ UnicodeString Mblem::call_server( const UnicodeString& instance ){
   //  LOG << "received json data:" << response.dump(2) << endl;
   string result = response["category"];
   //  LOG << "extracted result " << result << endl;
-  return TiCC::UnicodeFromUTF8(result);
+  return TiCC::UnicodeFromUTF8(result,_normalizer);
 }
 
 void Mblem::Classify( const icu::UnicodeString& uWord ){
@@ -587,7 +588,7 @@ void Mblem::Classify( const icu::UnicodeString& uWord ){
       vector<UnicodeString> edits = TiCC::split_at( partS, "+" );
       if ( edits.empty() ){
 	throw runtime_error( "invalid editstring: "
-			     + TiCC::UnicodeToUTF8(partS ) );
+			     + TiCC::UnicodeToUTF8(partS,_normalizer ) );
       }
       restag = edits[0]; // the first one is the POS tag
 
@@ -711,7 +712,7 @@ void Mblem::add_lemmas( const vector<folia::Word*>& wv,
     folia::KWargs args;
     args["set"] = getTagset();
     for ( const auto& lemma : fd.units[i].lemmas ){
-      args["class"] = TiCC::UnicodeToUTF8(lemma);
+      args["class"] = TiCC::UnicodeToUTF8(lemma,_normalizer);
       if ( textclass != "current" ){
 	args["textclass"] = textclass;
       }
diff --git a/src/mbma_brackets.cxx b/src/mbma_brackets.cxx
@@ -113,7 +113,7 @@ string toString( const Compound::Type& ct ){
   return "DEADLY";
 }
 
-Compound::Type stringToCompound( const string& s ){
+Compound::Type stringToCompound( const UnicodeString& s ){
   /// give the Compound::Type for this string
   if ( s == "NN" ){
     return Compound::Type::NN;
@@ -196,11 +196,11 @@ Compound::Type stringToCompound( const string& s ){
   else if ( s == "VV" ){
     return  Compound::Type::VV;
   }
-  else if ( s.empty() || s == "none" ){
+  else if ( s.isEmpty() || s == "none" ){
     return  Compound::Type::NONE;
   }
   else {
-    throw runtime_error( "no such compound:" + s );
+    throw runtime_error( "no such compound:" + TiCC::UnicodeToUTF8(s) );
   }
 }
 
@@ -532,7 +532,7 @@ Compound::Type construct( const vector<CLEX::Type>& tags ){
     \param tags a list of CLEX::Types
     \return a Compound::Type
   */
-  string s;
+  UnicodeString s;
   for ( const auto& t : tags ){
     s += toString( t );
   }
@@ -799,6 +799,7 @@ folia::Morpheme *BracketLeaf::createMorpheme( folia::Document *doc,
   */
   folia::Morpheme *result = 0;
   desc.remove();
+  TiCC::UnicodeNormalizer UN;
   int pos = _orig.indexOf( "^" );
   bool glue = ( pos != -1 );
   string m_class = toString( _status );
@@ -861,22 +862,22 @@ folia::Morpheme *BracketLeaf::createMorpheme( folia::Document *doc,
     args["set"] = Mbma::clex_tagset;
     if ( glue ){
       UnicodeString next_tag = _orig[pos+1];
-      args["class"] = TiCC::UnicodeToUTF8(next_tag);
+      args["class"] = TiCC::UnicodeToUTF8(next_tag,UN);
       desc = "[" + _morph + "]" + CLEX::get_tag_descr( CLEX::toCLEX(next_tag) );
       // spread the word upwards!
     }
     else {
-      args["class"] = toString( tag() );
+      args["class"] = TiCC::UnicodeToUTF8(toString(tag()), UN );
       desc = "[" + _morph + "]" + CLEX::get_tag_descr( tag() );
       // spread the word upwards!
       folia::KWargs fargs;
       fargs["subset"] = "structure";
       if ( tag() == CLEX::SPEC
 	   || tag() == CLEX::LET ){
-	fargs["class"] = TiCC::UnicodeToUTF8("[" + _morph + "]");
+	fargs["class"] = TiCC::UnicodeToUTF8("[" + _morph + "]",UN);
       }
       else {
-	fargs["class"] = TiCC::UnicodeToUTF8(desc);
+	fargs["class"] = TiCC::UnicodeToUTF8(desc,UN);
       }
 #pragma omp critical (foliaupdate)
       {
@@ -891,7 +892,7 @@ folia::Morpheme *BracketLeaf::createMorpheme( folia::Document *doc,
   else if ( _status == Status::PARTICLE ){
     folia::KWargs args;
     args["set"] = Mbma::clex_tagset;
-    args["class"] = toString( tag() );
+    args["class"] = TiCC::UnicodeToUTF8(toString( tag() ), UN );
 #pragma omp critical (foliaupdate)
     {
       result->addPosAnnotation( args );
@@ -916,7 +917,7 @@ folia::Morpheme *BracketLeaf::createMorpheme( folia::Document *doc,
 	UnicodeString d = CLEX::get_inflect_descr(inf);
 	if ( !d.isEmpty() ){
 	  // happens sometimes when there is fawlty data
-	  args["class"] = TiCC::UnicodeToUTF8(d);
+	  args["class"] = TiCC::UnicodeToUTF8(d,UN);
 	  desc += "/" + d;
 #pragma omp critical (foliaupdate)
 	  {
@@ -945,7 +946,7 @@ folia::Morpheme *BracketLeaf::createMorpheme( folia::Document *doc,
     // now we add the description as a feature
     folia::KWargs args;
     args["subset"] = "structure";
-    args["class"]  = TiCC::UnicodeToUTF8(desc);
+    args["class"]  = TiCC::UnicodeToUTF8(desc,UN);
 #pragma omp critical (foliaupdate)
     {
       result->add_child<folia::Feature>( args );
@@ -978,6 +979,7 @@ folia::Morpheme *BracketNest::createMorpheme( folia::Document *doc,
     \param cnt a counter for the number of handled morphemes
   */
   folia::Morpheme *result = 0;
+  TiCC::UnicodeNormalizer UN;
   folia::KWargs args;
   args["class"] = "complex";
   args["set"] = Mbma::mbma_tagset;
@@ -1000,7 +1002,7 @@ folia::Morpheme *BracketNest::createMorpheme( folia::Document *doc,
       if ( !it->original().isEmpty() ){
 	args.clear();
 	args["subset"] = "applied_rule";
-	args["class"] = TiCC::UnicodeToUTF8(it->original());
+	args["class"] = TiCC::UnicodeToUTF8(it->original(),UN);
 #pragma omp critical (foliaupdate)
 	{
 	  result->add_child<folia::Feature>( args );
@@ -1022,14 +1024,14 @@ folia::Morpheme *BracketNest::createMorpheme( folia::Document *doc,
   if ( desc.isEmpty() ){
     desc = "XYZ";
   }
-  args["class"] = TiCC::UnicodeToUTF8(desc);
+  args["class"] = TiCC::UnicodeToUTF8(desc,UN);
 #pragma omp critical (foliaupdate)
   {
     result->add_child<folia::Feature>( args );
   }
   args.clear();
   args["set"] = Mbma::clex_tagset;
-  args["class"] = toString( tag() );
+  args["class"] = TiCC::UnicodeToUTF8(toString( tag() ), UN);
   folia::PosAnnotation *pos = 0;
 #pragma omp critical (foliaupdate)
   {
diff --git a/src/mbma_mod.cxx b/src/mbma_mod.cxx
diff --git a/src/mbma_rule.cxx b/src/mbma_rule.cxx
diff --git a/src/mwu_chunker_mod.cxx b/src/mwu_chunker_mod.cxx
diff --git a/src/ner_tagger_mod.cxx b/src/ner_tagger_mod.cxx
diff --git a/src/tagger_base.cxx b/src/tagger_base.cxx
