Merge pull request #1088 from pilleye/pilleye/fix-unigram-crash

taku910 · web-flow · commit 273449044caa · 2025-02-27T00:14:58.000+09:00
fix crash in unigram model training
diff --git a/src/unigram_model_trainer.cc b/src/unigram_model_trainer.cc
@@ -273,7 +273,7 @@ TrainerModel::SentencePieces Trainer::MakeSeedSentencePiecesInternal() {
     for (node_int_type i = 0; i < node_num; ++i) {
       const node_int_type offset = SA[L[i]];
       const node_int_type len = D[i];
-      if (len <= 1) {
+      if (len <= 1 || offset >= array.size() || offset + len >= array.size()) {
         continue;
       }
       const char32 *begin = &array[offset];

Original file line number	Diff line number	Diff line change
`@@ -273,7 +273,7 @@ TrainerModel::SentencePieces Trainer::MakeSeedSentencePiecesInternal() {`
`273`	`273`	`for (node_int_type i = 0; i < node_num; ++i) {`
`274`	`274`	`const node_int_type offset = SA[L[i]];`
`275`	`275`	`const node_int_type len = D[i];`
`276`		`- if (len <= 1) {`
	`276`	`+ if (len <= 1 \|\| offset >= array.size() \|\| offset + len >= array.size()) {`
`277`	`277`	`continue;`
`278`	`278`	`}`
`279`	`279`	`const char32 *begin = &array[offset];`