How to build in android platform?

報告されたCMakeLists.txtとそのtarget_link_librariesエントリから始め、続いてAndroid向けにlibtokenizers_cpp.aと関連ライブラリがどのようにビルドおよびリンクされるかを調べます。リストされている未定義のtokenizers_*シンボルを使って不足している依存関係を追跡します。Androidのビルド手順と必要なリンク入力が明確になり、サンプルターゲットが正常にリンクできれば完了です。

int main() {
std::string tokenizer_json_path = "../tokenizer.json", path;
std::unique_ptrtokenizers::Tokenizer m_tokenizer;

path = LoadBytesFromFile(tokenizer_json_path);
cout<<"load file success !"<<endl;

m_tokenizer = tokenizers::Tokenizer::FromBlobJSON(path);

Here is my CMakeLists.txt info:
{

cmake_minimum_required(VERSION 3.18)

project(target)

set(CMAKE_CXX_STANDARD 17)
set(CMAKE_CXX_STANDARD_REQUIRED True)

aux_source_directory(. SRC_FILES)

link_directories(libs)

include_directories(include)

add_executable(target ${SRC_FILES})
target_link_libraries(target
${PROJECT_SOURCE_DIR}/libs/libsentencepiece.a
${PROJECT_SOURCE_DIR}/libs/libtokenizers_c.a
${PROJECT_SOURCE_DIR}/libs/libtokenizers_cpp.a
)

}

When I test, met the following issues:

/usr/bin/ld: ../libs/libtokenizers_cpp.a(huggingface_tokenizer.cc.o): in function tokenizers::Tokenizer::FromBlobJSON(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)': huggingface_tokenizer.cc:(.text+0x46): undefined reference to tokenizers_new_from_str'
/usr/bin/ld: ../libs/libtokenizers_cpp.a(huggingface_tokenizer.cc.o): in function tokenizers::Tokenizer::FromBlobByteLevelBPE(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&)': huggingface_tokenizer.cc:(.text+0x13e): undefined reference to byte_level_bpe_tokenizers_new_from_str'
/usr/bin/ld: ../libs/libtokenizers_cpp.a(huggingface_tokenizer.cc.o): in function tokenizers::HFTokenizer::~HFTokenizer()': huggingface_tokenizer.cc:(.text._ZN10tokenizers11HFTokenizerD2Ev[_ZN10tokenizers11HFTokenizerD5Ev]+0x37): undefined reference to tokenizers_free'
/usr/bin/ld: ../libs/libtokenizers_cpp.a(huggingface_tokenizer.cc.o): in function tokenizers::HFTokenizer::Encode(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool)': huggingface_tokenizer.cc:(.text._ZN10tokenizers11HFTokenizer6EncodeERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEb[_ZN10tokenizers11HFTokenizer6EncodeERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEb]+0x69): undefined reference to tokenizers_encode'
/usr/bin/ld: huggingface_tokenizer.cc:(.text._ZN10tokenizers11HFTokenizer6EncodeERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEb[_ZN10tokenizers11HFTokenizer6EncodeERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEb]+0xbc): undefined reference to tokenizers_free_encode_results' /usr/bin/ld: ../libs/libtokenizers_cpp.a(huggingface_tokenizer.cc.o): in function tokenizers::HFTokenizer::EncodeBatch(std::vector<std::__cxx11::basic_string<char, std::char_traits, std::allocator >, std::allocator<std::__cxx11::basic_string<char, std::char_traits, std::allocator > > > const&, bool)':
huggingface_tokenizer.cc:(.text._ZN10tokenizers11HFTokenizer11EncodeBatchERKSt6vectorINSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEESaIS7_EEb[_ZN10tokenizers11HFTokenizer11EncodeBatchERKSt6vectorINSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEESaIS7_EEb]+0x203): undefined reference to tokenizers_encode_batch' /usr/bin/ld: huggingface_tokenizer.cc:(.text._ZN10tokenizers11HFTokenizer11EncodeBatchERKSt6vectorINSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEESaIS7_EEb[_ZN10tokenizers11HFTokenizer11EncodeBatchERKSt6vectorINSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEESaIS7_EEb]+0x344): undefined reference to tokenizers_free_encode_results'
/usr/bin/ld: ../libs/libtokenizers_cpp.a(huggingface_tokenizer.cc.o): in function tokenizers::HFTokenizer::Decode[abi:cxx11](std::vector<int, std::allocator<int> > const&, bool)': huggingface_tokenizer.cc:(.text._ZN10tokenizers11HFTokenizer6DecodeB5cxx11ERKSt6vectorIiSaIiEEb[_ZN10tokenizers11HFTokenizer6DecodeB5cxx11ERKSt6vectorIiSaIiEEb]+0x62): undefined reference to tokenizers_decode'
/usr/bin/ld: huggingface_tokenizer.cc:(.text._ZN10tokenizers11HFTokenizer6DecodeB5cxx11ERKSt6vectorIiSaIiEEb[_ZN10tokenizers11HFTokenizer6DecodeB5cxx11ERKSt6vectorIiSaIiEEb]+0x7d): undefined reference to tokenizers_get_decode_str' /usr/bin/ld: ../libs/libtokenizers_cpp.a(huggingface_tokenizer.cc.o): in function tokenizers::HFTokenizer::GetVocabSize()':
huggingface_tokenizer.cc:(.text._ZN10tokenizers11HFTokenizer12GetVocabSizeEv[_ZN10tokenizers11HFTokenizer12GetVocabSizeEv]+0x32): undefined reference to tokenizers_get_vocab_size' /usr/bin/ld: ../libs/libtokenizers_cpp.a(huggingface_tokenizer.cc.o): in function tokenizers::HFTokenizer::IdToTokenabi:cxx11':
huggingface_tokenizer.cc:(.text._ZN10tokenizers11HFTokenizer9IdToTokenB5cxx11Ei[_ZN10tokenizers11HFTokenizer9IdToTokenB5cxx11Ei]+0x3e): undefined reference to tokenizers_id_to_token' /usr/bin/ld: ../libs/libtokenizers_cpp.a(huggingface_tokenizer.cc.o): in function tokenizers::HFTokenizer::TokenToId(std::__cxx11::basic_string<char, std::char_traits, std::allocator > const&)':
huggingface_tokenizer.cc:(.text._ZN10tokenizers11HFTokenizer9TokenToIdERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEE[_ZN10tokenizers11HFTokenizer9TokenToIdERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEE]+0x58): undefined reference to `tokenizers_token_to_id'
collect2: error: ld returned 1 exit status
make[2]: *** [CMakeFiles/target.dir/build.make:116: target] Error 1
make[1]: *** [CMakeFiles/Makefile2:83: CMakeFiles/target.dir/all] Error 2