Skip to content

compute image start_idx after audio expansion #1398

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
shihaobai merged 2 commits into from
Jul 22, 2026
+27 −26
Merged

compute image start_idx after audio expansion #1398

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
dcfc265
compute image start_idx after audio expansion
WANDY666 Jul 20, 2026
8ee9a96
Simplify
WANDY666 Jul 21, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 27 additions & 26 deletions lightllm/models/qwen3_omni_moe_thinker/model.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,59 +89,60 @@ def encode(self, prompt, multimodal_params: MultimodalParams = None, **kwargs):

# <img><image_pad></img> -> <img></img>
origin_ids = [token for token in origin_ids if token not in (self.image_token_id, self.audio_token_id)]
# <img></img> --> <img>id,id+1...id+num</img>

# audio
input_ids = []
image_id = 0
audio_id = 0
start_idx = 0
while True:
try:
start_idx = origin_ids.index(self.image_start_id)
start_idx = origin_ids.index(self.audio_start_id)
if start_idx + 1 >= len(origin_ids):
break
if origin_ids[start_idx + 1] == self.image_end_id:
if origin_ids[start_idx + 1] == self.audio_end_id:
input_ids.extend(origin_ids[: start_idx + 1])
token_id = multimodal_params.images[image_id].token_id
token_num = multimodal_params.images[image_id].token_num
multimodal_params.images[image_id].start_idx = len(input_ids)
token_id = multimodal_params.audios[audio_id].token_id
token_num = multimodal_params.audios[audio_id].token_num
input_ids.extend(range(token_id, token_id + token_num))
input_ids.append(self.image_end_id)
input_ids.append(self.audio_end_id)
origin_ids = origin_ids[start_idx + 2 :]
image_id += 1
audio_id += 1
else:
raise ValueError("image token error")
raise ValueError("audio token error")
except ValueError:
break
if multimodal_params:
image_cnt = len(multimodal_params.images)
if image_cnt != image_id:
raise ValueError(image_cnt == image_id, f"invalid image tag num: {image_cnt} vs {image_id}!")
audio_cnt = len(multimodal_params.audios)
if audio_cnt != audio_id:
raise ValueError(audio_cnt == audio_id, f"invalid audio tag num: {audio_cnt} vs {audio_id}!")
input_ids.extend(origin_ids)

# audio
# <img></img> --> <img>id,id+1...id+num</img>
origin_ids = input_ids
input_ids = []
audio_id = 0
start_idx = 0
image_id = 0
while True:
try:
start_idx = origin_ids.index(self.audio_start_id)
start_idx = origin_ids.index(self.image_start_id)
if start_idx + 1 >= len(origin_ids):
break
if origin_ids[start_idx + 1] == self.audio_end_id:
if origin_ids[start_idx + 1] == self.image_end_id:
input_ids.extend(origin_ids[: start_idx + 1])
token_id = multimodal_params.audios[audio_id].token_id
token_num = multimodal_params.audios[audio_id].token_num
token_id = multimodal_params.images[image_id].token_id
token_num = multimodal_params.images[image_id].token_num
multimodal_params.images[image_id].start_idx = len(input_ids)
input_ids.extend(range(token_id, token_id + token_num))
input_ids.append(self.audio_end_id)
input_ids.append(self.image_end_id)
origin_ids = origin_ids[start_idx + 2 :]
audio_id += 1
image_id += 1
else:
raise ValueError("audio token error")
raise ValueError("image token error")
except ValueError:
break
if multimodal_params:
audio_cnt = len(multimodal_params.audios)
if audio_cnt != audio_id:
raise ValueError(audio_cnt == audio_id, f"invalid audio tag num: {audio_cnt} vs {audio_id}!")
image_cnt = len(multimodal_params.images)
if image_cnt != image_id:
raise ValueError(image_cnt == image_id, f"invalid image tag num: {image_cnt} vs {image_id}!")
input_ids.extend(origin_ids)

return input_ids
Expand Down
Loading