Index

_

__init__() (flagevalmm.dataset.data_preprocessor.DataPreprocessor method) (flagevalmm.eval.ServerWrapper method) (flagevalmm.evaluator.base_evaluator.BaseEvaluator method) (flagevalmm.evaluator.base_evaluator.QuestionMapping method) (flagevalmm.evaluator.extract_evaluator.ExtractEvaluator method) (flagevalmm.evaluator.retrieval_evaluator.RetrievalEvaluator method) (flagevalmm.models.model_cache.ModelCache method) (flagevalmm.server.model_server.ModelServer method) (flagevalmm.server.ModelServer method) (flagevalmm.server.server_dataset.ServerDataset method) (flagevalmm.server.ServerDataset method)

A

aggregate_multi_inference_results() (flagevalmm.evaluator.base_evaluator.BaseEvaluator method)

B

BaseEvaluator (class in flagevalmm.evaluator.base_evaluator)

C

cal_accuracy() (flagevalmm.evaluator.base_evaluator.BaseEvaluator method) (flagevalmm.evaluator.extract_evaluator.ExtractEvaluator method) (flagevalmm.evaluator.mmmu_dataset_evaluator.MmmuEvaluator method) calculate_hash() (in module flagevalmm.models.model_cache) check_is_number() (in module flagevalmm.evaluator.mmmu_dataset_evaluator) cleanup() (flagevalmm.eval.ServerWrapper method) (flagevalmm.server.model_server.ModelServer method) (flagevalmm.server.ModelServer method)

clear() (flagevalmm.models.model_cache.ModelCache method) close() (flagevalmm.models.model_cache.ModelCache method) ColorFormatter (class in flagevalmm.common.logger) compare_answer() (flagevalmm.evaluator.extract_evaluator.ExtractEvaluator method) convert_circled_numbers() (in module flagevalmm.evaluator.pre_process)

D

DataPreprocessor (class in flagevalmm.dataset.data_preprocessor)

default_collate_fn() (in module flagevalmm.server.utils) delete() (flagevalmm.models.model_cache.ModelCache method)

E

eval_open() (in module flagevalmm.evaluator.mmmu_dataset_evaluator) evaluate_by_llm() (flagevalmm.evaluator.base_evaluator.BaseEvaluator method) evaluate_fill_blank_by_rule() (flagevalmm.evaluator.base_evaluator.BaseEvaluator method) evaluate_multiple_choice() (flagevalmm.evaluator.base_evaluator.BaseEvaluator method) evaluate_multiple_response() (flagevalmm.evaluator.base_evaluator.BaseEvaluator method) evaluate_only() (in module flagevalmm.eval)

exists() (flagevalmm.models.model_cache.ModelCache method) expand_multi_inference_predictions() (flagevalmm.evaluator.base_evaluator.BaseEvaluator method) extract_answer_by_llm() (flagevalmm.evaluator.extract_evaluator.ExtractEvaluator method) extract_judgement_result() (flagevalmm.evaluator.base_evaluator.BaseEvaluator method) extract_numbers() (in module flagevalmm.evaluator.mmmu_dataset_evaluator) ExtractEvaluator (class in flagevalmm.evaluator.extract_evaluator)

F

filter_finished_tasks() (flagevalmm.eval.ServerWrapper method) filter_rejected() (flagevalmm.evaluator.base_evaluator.BaseEvaluator method) flagevalmm module flagevalmm.common module flagevalmm.common.const module flagevalmm.common.logger module flagevalmm.dataset.data_preprocessor module flagevalmm.eval module flagevalmm.evaluator.base_evaluator module flagevalmm.evaluator.extract_evaluator module

flagevalmm.evaluator.mmmu_dataset_evaluator module flagevalmm.evaluator.pre_process module flagevalmm.evaluator.retrieval_evaluator module flagevalmm.models.model_cache module flagevalmm.registry module flagevalmm.server module flagevalmm.server.model_server module flagevalmm.server.server_dataset module flagevalmm.server.utils module format() (flagevalmm.common.logger.ColorFormatter method)

G

get() (flagevalmm.models.model_cache.ModelCache method) get_data() (flagevalmm.server.server_dataset.ServerDataset method) (flagevalmm.server.ServerDataset method) (in module flagevalmm.server.utils) get_eval_func() (flagevalmm.evaluator.base_evaluator.BaseEvaluator method) get_flagscale_cmd() (flagevalmm.server.model_server.ModelServer method) (flagevalmm.server.ModelServer method) get_lmdeploy_cmd() (flagevalmm.server.model_server.ModelServer method) (flagevalmm.server.ModelServer method) get_logger() (in module flagevalmm.common) (in module flagevalmm.common.logger)

get_meta() (in module flagevalmm.server.utils) get_random_port() (in module flagevalmm.server.utils) get_retrieval_data() (in module flagevalmm.server.utils) get_sglang_cmd() (flagevalmm.server.model_server.ModelServer method) (flagevalmm.server.ModelServer method) get_task_info() (in module flagevalmm.server.utils) get_vllm_cmd() (flagevalmm.server.model_server.ModelServer method) (flagevalmm.server.ModelServer method) grade_by_simpleqa() (flagevalmm.evaluator.extract_evaluator.ExtractEvaluator method)

H

has_multi_inference() (flagevalmm.evaluator.base_evaluator.BaseEvaluator method)

I

i2t() (in module flagevalmm.evaluator.retrieval_evaluator) inference_index (flagevalmm.evaluator.base_evaluator.QuestionMapping attribute) insert() (flagevalmm.models.model_cache.ModelCache method)

is_multi_inference (flagevalmm.evaluator.base_evaluator.QuestionMapping attribute) is_port_occupied() (in module flagevalmm.server.utils) is_server_running() (flagevalmm.eval.ServerWrapper method)

J

json_save() (in module flagevalmm.evaluator.retrieval_evaluator)

L

launch_server() (flagevalmm.server.model_server.ModelServer method) (flagevalmm.server.ModelServer method)

load_pil_image() (in module flagevalmm.server.utils)

M

maybe_clean_answer() (flagevalmm.evaluator.base_evaluator.BaseEvaluator method) maybe_launch_evaluation_server() (flagevalmm.eval.ServerWrapper method) maybe_register_class() (in module flagevalmm.server.utils) merge_args() (in module flagevalmm.server.utils) MmmuEvaluator (class in flagevalmm.evaluator.mmmu_dataset_evaluator) ModelCache (class in flagevalmm.models.model_cache) ModelServer (class in flagevalmm.server) (class in flagevalmm.server.model_server) module flagevalmm flagevalmm.common flagevalmm.common.const flagevalmm.common.logger flagevalmm.dataset.data_preprocessor flagevalmm.eval flagevalmm.evaluator.base_evaluator flagevalmm.evaluator.extract_evaluator flagevalmm.evaluator.mmmu_dataset_evaluator flagevalmm.evaluator.pre_process flagevalmm.evaluator.retrieval_evaluator flagevalmm.models.model_cache flagevalmm.registry flagevalmm.server flagevalmm.server.model_server flagevalmm.server.server_dataset flagevalmm.server.utils

N

normalize_str() (in module flagevalmm.evaluator.mmmu_dataset_evaluator)

normalize_string() (in module flagevalmm.evaluator.pre_process)

O

original_question_id (flagevalmm.evaluator.base_evaluator.QuestionMapping attribute)

P

parse_args() (in module flagevalmm.dataset.data_preprocessor) (in module flagevalmm.server.utils) parse_config() (flagevalmm.dataset.data_preprocessor.DataPreprocessor method) parse_open_response() (in module flagevalmm.evaluator.mmmu_dataset_evaluator) process() (flagevalmm.dataset.data_preprocessor.DataPreprocessor method) (flagevalmm.evaluator.base_evaluator.BaseEvaluator method) (flagevalmm.evaluator.extract_evaluator.ExtractEvaluator method) (flagevalmm.evaluator.retrieval_evaluator.RetrievalEvaluator method)

process_images_symbol() (in module flagevalmm.server.utils) process_multiple_choice() (in module flagevalmm.evaluator.pre_process) process_single_prediction() (flagevalmm.evaluator.extract_evaluator.ExtractEvaluator method)

Q

QuestionMapping (class in flagevalmm.evaluator.base_evaluator)

R

remove_special_characters() (in module flagevalmm.evaluator.pre_process) remove_unit() (in module flagevalmm.evaluator.pre_process)

RetrievalEvaluator (class in flagevalmm.evaluator.retrieval_evaluator) run() (in module flagevalmm.eval) run_model_adapter() (flagevalmm.eval.ServerWrapper method)

S

save() (flagevalmm.evaluator.base_evaluator.BaseEvaluator method) ServerDataset (class in flagevalmm.server) (class in flagevalmm.server.server_dataset) ServerWrapper (class in flagevalmm.eval) start() (flagevalmm.eval.ServerWrapper method)

statistics_tokens() (flagevalmm.evaluator.base_evaluator.BaseEvaluator method) stop() (flagevalmm.server.model_server.ModelServer method) (flagevalmm.server.ModelServer method) strip_answer() (in module flagevalmm.evaluator.pre_process) submit() (in module flagevalmm.server.utils)

T

t2i() (in module flagevalmm.evaluator.retrieval_evaluator)

total_inferences (flagevalmm.evaluator.base_evaluator.QuestionMapping attribute)

U

update_cfg_from_args() (in module flagevalmm.eval)