File size: 2,524 Bytes
59fa730
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
{
  "model": "Qwen/Qwen3-Reranker-0.6B",
  "seq_len": 512,
  "dtype": "float16",
  "yes_id": 9693,
  "no_id": 2152,
  "pad_token_id": 151643,
  "padding_side": "right",
  "prefix": "<|im_start|>system\nJudge whether the Document meets the requirements based on the Query and the Instruct provided. Note that the answer can only be \"yes\" or \"no\".<|im_end|>\n<|im_start|>user\n",
  "suffix": "<|im_end|>\n<|im_start|>assistant\n<think>\n\n</think>\n\n",
  "default_instruction": "Given a web search query, retrieve relevant passages that answer the query",
  "output": "probs [1,2] = softmax([no, yes]); relevance = probs[1] = P(yes)",
  "pairs": {
    "rel_capital": {
      "relevant": true,
      "query": "What is the capital of Japan?",
      "doc": "Tokyo is the capital and largest city of Japan."
    },
    "rel_beesting": {
      "relevant": true,
      "query": "How do I treat a bee sting?",
      "doc": "Remove the stinger, wash with soap and water, then apply a cold pack to reduce swelling."
    },
    "rel_fuji_ja": {
      "relevant": true,
      "query": "富士山の高さはどのくらいですか?",
      "doc": "富士山は標高3,776メートルで、日本で最も高い山です。"
    },
    "irr_capital": {
      "relevant": false,
      "query": "What is the capital of Japan?",
      "doc": "Python is the most widely used programming language for machine learning."
    },
    "irr_beesting": {
      "relevant": false,
      "query": "How do I treat a bee sting?",
      "doc": "Tokyo is the capital and largest city of Japan."
    },
    "irr_fuji_ja": {
      "relevant": false,
      "query": "富士山の高さはどのくらいですか?",
      "doc": "The recipe calls for two eggs and a cup of flour."
    }
  },
  "scores": {
    "rel_capital": 0.9937748908996582,
    "rel_beesting": 0.9814825654029846,
    "rel_fuji_ja": 0.9997767806053162,
    "irr_capital": 8.651458301756065e-06,
    "irr_beesting": 3.616727553890087e-05,
    "irr_fuji_ja": 5.813539701193804e-06
  },
  "official_scores": {
    "rel_capital": 0.9937747716903687,
    "rel_beesting": 0.9814824461936951,
    "rel_fuji_ja": 0.9997767806053162,
    "irr_capital": 8.651466487208381e-06,
    "irr_beesting": 3.616731191868894e-05,
    "irr_fuji_ja": 5.813562438561348e-06
  },
  "rank_groups": {
    "capital": [
      "rel_capital",
      "irr_capital"
    ],
    "beesting": [
      "rel_beesting",
      "irr_beesting"
    ],
    "fuji_ja": [
      "rel_fuji_ja",
      "irr_fuji_ja"
    ]
  }
}