fix crash in unigram model training

pilleye · pilleye · commit d19ac45c9196 · 2025-02-13T16:15:42.000-05:00
cherry pick 0fe7add, was blasted away in pilleye@53de765
diff --git a/src/unigram_model_trainer.cc b/src/unigram_model_trainer.cc
@@ -273,7 +273,7 @@ TrainerModel::SentencePieces Trainer::MakeSeedSentencePiecesInternal() {
     for (node_int_type i = 0; i < node_num; ++i) {
       const node_int_type offset = SA[L[i]];
       const node_int_type len = D[i];
-      if (len <= 1) {
+      if (len <= 1 || offset >= array.size() || offset + len >= array.size()) {
         continue;
       }
       const char32 *begin = &array[offset];

Original file line number	Diff line number	Diff line change
`@@ -273,7 +273,7 @@ TrainerModel::SentencePieces Trainer::MakeSeedSentencePiecesInternal() {`
`273`	`273`	`for (node_int_type i = 0; i < node_num; ++i) {`
`274`	`274`	`const node_int_type offset = SA[L[i]];`
`275`	`275`	`const node_int_type len = D[i];`
`276`		`- if (len <= 1) {`
	`276`	`+ if (len <= 1 \|\| offset >= array.size() \|\| offset + len >= array.size()) {`
`277`	`277`	`continue;`
`278`	`278`	`}`
`279`	`279`	`const char32 *begin = &array[offset];`