diff --git a/lightllm/models/qwen3_omni_moe_thinker/model.py b/lightllm/models/qwen3_omni_moe_thinker/model.py index 45fb16f70..e4ea3ccc3 100644 --- a/lightllm/models/qwen3_omni_moe_thinker/model.py +++ b/lightllm/models/qwen3_omni_moe_thinker/model.py @@ -89,59 +89,60 @@ def encode(self, prompt, multimodal_params: MultimodalParams = None, **kwargs): # -> origin_ids = [token for token in origin_ids if token not in (self.image_token_id, self.audio_token_id)] - # --> id,id+1...id+num + + # audio input_ids = [] - image_id = 0 + audio_id = 0 + start_idx = 0 while True: try: - start_idx = origin_ids.index(self.image_start_id) + start_idx = origin_ids.index(self.audio_start_id) if start_idx + 1 >= len(origin_ids): break - if origin_ids[start_idx + 1] == self.image_end_id: + if origin_ids[start_idx + 1] == self.audio_end_id: input_ids.extend(origin_ids[: start_idx + 1]) - token_id = multimodal_params.images[image_id].token_id - token_num = multimodal_params.images[image_id].token_num - multimodal_params.images[image_id].start_idx = len(input_ids) + token_id = multimodal_params.audios[audio_id].token_id + token_num = multimodal_params.audios[audio_id].token_num input_ids.extend(range(token_id, token_id + token_num)) - input_ids.append(self.image_end_id) + input_ids.append(self.audio_end_id) origin_ids = origin_ids[start_idx + 2 :] - image_id += 1 + audio_id += 1 else: - raise ValueError("image token error") + raise ValueError("audio token error") except ValueError: break if multimodal_params: - image_cnt = len(multimodal_params.images) - if image_cnt != image_id: - raise ValueError(image_cnt == image_id, f"invalid image tag num: {image_cnt} vs {image_id}!") + audio_cnt = len(multimodal_params.audios) + if audio_cnt != audio_id: + raise ValueError(audio_cnt == audio_id, f"invalid audio tag num: {audio_cnt} vs {audio_id}!") input_ids.extend(origin_ids) - # audio + # --> id,id+1...id+num origin_ids = input_ids input_ids = [] - audio_id = 0 - start_idx = 0 + image_id = 0 while True: try: - start_idx = origin_ids.index(self.audio_start_id) + start_idx = origin_ids.index(self.image_start_id) if start_idx + 1 >= len(origin_ids): break - if origin_ids[start_idx + 1] == self.audio_end_id: + if origin_ids[start_idx + 1] == self.image_end_id: input_ids.extend(origin_ids[: start_idx + 1]) - token_id = multimodal_params.audios[audio_id].token_id - token_num = multimodal_params.audios[audio_id].token_num + token_id = multimodal_params.images[image_id].token_id + token_num = multimodal_params.images[image_id].token_num + multimodal_params.images[image_id].start_idx = len(input_ids) input_ids.extend(range(token_id, token_id + token_num)) - input_ids.append(self.audio_end_id) + input_ids.append(self.image_end_id) origin_ids = origin_ids[start_idx + 2 :] - audio_id += 1 + image_id += 1 else: - raise ValueError("audio token error") + raise ValueError("image token error") except ValueError: break if multimodal_params: - audio_cnt = len(multimodal_params.audios) - if audio_cnt != audio_id: - raise ValueError(audio_cnt == audio_id, f"invalid audio tag num: {audio_cnt} vs {audio_id}!") + image_cnt = len(multimodal_params.images) + if image_cnt != image_id: + raise ValueError(image_cnt == image_id, f"invalid image tag num: {image_cnt} vs {image_id}!") input_ids.extend(origin_ids) return input_ids