mirror of
https://github.com/SYSTRAN/faster-whisper.git
synced 2026-01-08 13:14:00 -05:00
14ba1051f3
* Fix: Prevent <|nocaptions|> tokens in BatchedInferencePipeline
- Add nocaptions component tokens [1771, 496, 9799] to suppress_tokens list
- Add segment filtering to remove any remaining <|nocaptions|> segments
- Resolves issue where BatchedInferencePipeline would generate malformed
special tokens during periods of silence or low-confidence transcription
- Includes comprehensive tests to verify the fix
The issue occurred because while bracket tokens ('<', '|', '>') were
already suppressed, the content tokens ('no', 'ca', 'ptions') were not,
leading to partial token generation that formed complete <|nocaptions|>
tags in the output.
Files changed:
- faster_whisper/transcribe.py: Core fix implementation
- test_nocaptions_comprehensive.py: Comprehensive test suite
- tests/test_nocaptions_fix.py: Unit tests
* removed
* Fix: Prevent <|nocaptions|> tokens in BatchedInferencePipeline
* Fix: Implement proper <|nocaptions|> token suppression using single token approach
* ci: trigger tests
* fix: remove trailing whitespace from blank lines
* Update faster_whisper/transcribe.py
Co-authored-by: Mahmoud Ashraf <hassouna97.ma@gmail.com>
* Update faster_whisper/tokenizer.py
Co-authored-by: Mahmoud Ashraf <hassouna97.ma@gmail.com>
* Update faster_whisper/tokenizer.py
Co-authored-by: Mahmoud Ashraf <hassouna97.ma@gmail.com>
* Rename no_speech to no_captions in tokenizer
* nocaptions has been renamed to nospeech
* break line
* line break
* Refactor no_speech method for improved readability by adjusting line breaks
---------
Co-authored-by: Mahmoud Ashraf <hassouna97.ma@gmail.com>
122 lines
2.2 KiB
Python
122 lines
2.2 KiB
Python
from faster_whisper import WhisperModel
|
|
from faster_whisper.tokenizer import Tokenizer
|
|
from faster_whisper.transcribe import get_suppressed_tokens
|
|
|
|
|
|
def test_suppressed_tokens_minus_1():
|
|
model = WhisperModel("tiny.en")
|
|
|
|
tokenizer = Tokenizer(model.hf_tokenizer, False)
|
|
tokens = get_suppressed_tokens(tokenizer, [-1])
|
|
assert tokens == (
|
|
1,
|
|
2,
|
|
7,
|
|
8,
|
|
9,
|
|
10,
|
|
14,
|
|
25,
|
|
26,
|
|
27,
|
|
28,
|
|
29,
|
|
31,
|
|
58,
|
|
59,
|
|
60,
|
|
61,
|
|
62,
|
|
63,
|
|
90,
|
|
91,
|
|
92,
|
|
93,
|
|
357,
|
|
366,
|
|
438,
|
|
532,
|
|
685,
|
|
705,
|
|
796,
|
|
930,
|
|
1058,
|
|
1220,
|
|
1267,
|
|
1279,
|
|
1303,
|
|
1343,
|
|
1377,
|
|
1391,
|
|
1635,
|
|
1782,
|
|
1875,
|
|
2162,
|
|
2361,
|
|
2488,
|
|
3467,
|
|
4008,
|
|
4211,
|
|
4600,
|
|
4808,
|
|
5299,
|
|
5855,
|
|
6329,
|
|
7203,
|
|
9609,
|
|
9959,
|
|
10563,
|
|
10786,
|
|
11420,
|
|
11709,
|
|
11907,
|
|
13163,
|
|
13697,
|
|
13700,
|
|
14808,
|
|
15306,
|
|
16410,
|
|
16791,
|
|
17992,
|
|
19203,
|
|
19510,
|
|
20724,
|
|
22305,
|
|
22935,
|
|
27007,
|
|
30109,
|
|
30420,
|
|
33409,
|
|
34949,
|
|
40283,
|
|
40493,
|
|
40549,
|
|
47282,
|
|
49146,
|
|
50257,
|
|
50357,
|
|
50358,
|
|
50359,
|
|
50360,
|
|
50361,
|
|
)
|
|
|
|
|
|
def test_suppressed_tokens_minus_value():
|
|
model = WhisperModel("tiny.en")
|
|
|
|
tokenizer = Tokenizer(model.hf_tokenizer, False)
|
|
tokens = get_suppressed_tokens(tokenizer, [13])
|
|
assert tokens == (13, 50257, 50357, 50358, 50359, 50360, 50361)
|
|
|
|
|
|
def test_split_on_unicode():
|
|
model = WhisperModel("tiny")
|
|
tokenizer = Tokenizer(model.hf_tokenizer, False)
|
|
|
|
tokens = [8404, 871, 287, 6, 246, 526, 3210, 20378]
|
|
words, word_tokens = tokenizer.split_tokens_on_unicode(tokens)
|
|
|
|
assert words == [" elle", " est", " l", "'", "\ufffd", "é", "rit", "oire"]
|
|
assert word_tokens == [[8404], [871], [287], [6], [246], [526], [3210], [20378]]
|