axs2mlperf/base_small_llm_loadgen_experiment/data_axs.json at master · krai/axs2mlperf · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
{
    "_parent_entries": [ [ "^", "byname", "base_loadgen_experiment" ] ],

    "transformers_query": [ "python_package", "package_name=transformers", ["desired_python_version", ["^", "kernel_python_major_dot_minor"]] ],

    "_BEFORE_CODE_LOADING": [ "^^", "execute", [[
        [ "get_kernel" ],
        [ "byquery", [[ "^^", "get", "transformers_query" ]] ],
        [ "use" ]
    ]] ],

    "desired_python_version": "3.10",

    "mlperf_inference_git_entry": [ "^", "byquery", "git_repo,repo_name=mlperf_inference_git" ],

    "abs_script_path": [ "^^", "execute", [[
        [ "get", "mlperf_inference_git_entry" ],
        [ "get_path_of", "llama3_1_8b_cnndm_accuracy_script" ]
    ]] ],

    "accuracy_log_path": ["^^", "get_path", "mlperf_log_accuracy.json"],

    "dataset_name": "cnndm",
    "model_family": "llama3_1",
    "model_variant": "8b",

    "dataset_query": [ "downloaded", [ "^^", "substitute", "dataset_name=#{dataset_name}#,model_family=#{model_family}#,variant=#{model_variant}#" ]],
    "dataset_entry": [ "^", "byquery", [[ "^^", "get", "dataset_query" ]], {}, ["dataset_query"] ],

    "dataset_path": [ "^^", "execute", [[
        [ "get", "dataset_entry" ],
        [ "get_path" ],
        [ "__add__", "/cnn_eval.json" ]
    ]] ],

    "checkpoint_path_query": [ "^^", "substitute", "downloaded,hf_tokeniser,model_family=#{model_family}#,variant=#{model_variant}#" ],
    "checkpoint_path": [ "^^", "execute", [[
        [ "get_kernel" ],
        [ "byquery", [[ "^^", "get", "checkpoint_path_query" ]] ],
        [ "get_path" ]
    ]] ],

    "accuracy_log_dtype": "int32",

    "hf_token": "",

    "extract_accuracy_report": [ "^^", "execute", [[
        [ "get_kernel" ],
        [ "byname", "python_script" ],
        [ "run", [], {
                "python_deps": [
		    [ "AS^IS", "^^", "python_sync_pip_package", "python_package,package_name=torchvision,package_version=0.20.1" ],
                    [ "AS^IS", "^^", "python_sync_pip_package", "python_package,package_name=accelerate,package_version=0.21.0" ],
                    [ "AS^IS", "^^", "python_sync_pip_package", "python_package,package_name=pybind11,package_version=2.10.4" ],
                    [ "AS^IS", "^^", "python_sync_pip_package", "python_package,package_name=transformers,package_version=4.46.2" ],
                    [ "AS^IS", "^^", "python_sync_pip_package", "python_package,package_name=nltk,package_version=3.8.1" ],
                    [ "AS^IS", "^^", "python_sync_pip_package", "python_package,package_name=rouge_score,package_version=0.1.2" ],
                    [ "AS^IS", "^^", "python_sync_pip_package", "python_package,package_name=sentencepiece,package_version=0.2.0" ],
                    [ "AS^IS", "^^", "python_sync_pip_package", "python_package,package_name=absl-py,package_version=1.4.0" ],
                    [ "AS^IS", "^^", "python_sync_pip_package", "python_package,package_name=evaluate,package_version=0.4.0" ],
                    [ "AS^IS", "^^", "python_sync_pip_package", "python_package,package_name=vllm,package_version=0.6.3" ]
                ],
                "abs_script_path": ["^^", "get", "abs_script_path"],
                "script_extra_params": [ "^^", "substitute", "--mlperf-accuracy-file #{accuracy_log_path}# --dataset-file #{dataset_path}# --dtype #{accuracy_log_dtype}#" ],
                "desired_python_version": ["^", "kernel_python_major_dot_minor"],
                "capture_output": true,
                "extra_env": { "HUGGING_FACE_HUB_TOKEN": [ "^^", "substitute", "#{hf_token}#" ] }
            } ],
        0,
        [ "func", [ "ufun.rematch", "(\\{.*\\})" ] ],
        0,
        [ "denumpify_dict" ],
        0,
        [ "func", "str" ]
    ]], {} ],

    "accuracy_dict_full": [ "^^", "execute", [[
        ["get", "accuracy_report" ],
        0,
        [ "func", "eval" ]
     ]], {} ],
    "accuracy_dict": [ "^^", "get_accuracy_dict" ],
    "rouge1": [ "^^" , "dig","accuracy_dict.rouge1" ],
    "rouge2": [ "^^" , "dig","accuracy_dict.rouge2" ],
    "rougeL": [ "^^" , "dig","accuracy_dict.rougeL" ],
    "rougeLsum": [ "^^" , "dig","accuracy_dict.rougeLsum" ],
    "gen_len": [ "^^" , "dig","accuracy_dict.gen_len" ],
    "gen_num": [ "^^" , "dig","accuracy_dict.gen_num" ],
    "tokens_per_sample": [ "^^" , "dig","accuracy_dict.tokens_per_sample" ],

    "tokenised_accuracy_log_path": [ "^^", "get_path", "mlperf_log_accuracy.json" ],
    "output_log_path": [ "^^", "get_path", "detokenised_mlperf_log.json" ],

    "detokenised_log": [ "^^", "detokenise" ]
}