343 lines
8.6 KiB
JSON
343 lines
8.6 KiB
JSON
{
|
|
"revision": "15852e8c16360a2fea060d615a32b45270f8a8fc",
|
|
"tokenizer_sha256": "5f9e4d4901a92b997e463c1f46055088b6cca5ca61a6522d1b9f64c4bb81cb42",
|
|
"samples": [
|
|
{
|
|
"id": "cal-zh-1",
|
|
"split": "calibration",
|
|
"language": "zh",
|
|
"text": "\u6e05\u6668\uff0c\u7814\u7a76\u5458\u6253\u5f00\u5b9e\u9a8c\u8bb0\u5f55\u672c\uff0c\u68c0\u67e5\u6628\u5929\u4fdd\u5b58\u7684\u6570\u636e\u3002\u4ed6\u5148\u786e\u8ba4\u6587\u4ef6\u5b8c\u6574\uff0c\u518d\u6bd4\u8f83\u4e24\u7ec4\u89c2\u6d4b\u7ed3\u679c\u3002\u4e3a\u4e86\u907f\u514d\u5076\u7136\u8bef\u5dee\uff0c\u4ed6\u4e3a\u6bcf\u4e00\u6b65\u7559\u4e0b\u65f6\u95f4\u548c\u6761\u4ef6\uff0c\u5e76\u5c06\u5c1a\u672a\u9a8c\u8bc1\u7684\u63a8\u6d4b\u5355\u72ec\u8bb0\u5f55\u3002",
|
|
"text_sha256": "ac4ed82b8c201d572f5085c76c15cff31d700106aa46b18146da8ca14db299a1",
|
|
"full_token_count": 43,
|
|
"token_ids": [
|
|
111164,
|
|
3709,
|
|
112124,
|
|
103704,
|
|
98631,
|
|
98151,
|
|
95879,
|
|
3709,
|
|
97552,
|
|
103892,
|
|
101754,
|
|
104027,
|
|
1710,
|
|
95948,
|
|
96218,
|
|
101632,
|
|
97380,
|
|
99492,
|
|
3709,
|
|
96390,
|
|
97016,
|
|
120408,
|
|
112769,
|
|
97989,
|
|
1710,
|
|
120190,
|
|
112614,
|
|
111162,
|
|
3709,
|
|
125067,
|
|
121735,
|
|
108648
|
|
]
|
|
},
|
|
{
|
|
"id": "cal-zh-2",
|
|
"split": "calibration",
|
|
"language": "zh",
|
|
"text": "\u4e00\u8f86\u5217\u8f66\u4ece\u5c71\u95f4\u9a76\u5411\u6d77\u8fb9\uff0c\u7ecf\u8fc7\u6751\u5e84\u3001\u6865\u6881\u548c\u5f00\u9614\u7684\u7530\u91ce\u3002\u65c5\u5ba2\u671b\u7740\u7a97\u5916\u53d8\u5316\u7684\u666f\u8272\uff0c\u8ba1\u5212\u62b5\u8fbe\u540e\u5148\u6b65\u884c\u5230\u65c5\u9986\uff0c\u518d\u53bb\u5e02\u573a\u8d2d\u4e70\u665a\u9910\u9700\u8981\u7684\u98df\u6750\u3002\u5929\u6c14\u9884\u62a5\u8bf4\u508d\u665a\u53ef\u80fd\u4e0b\u96e8\u3002",
|
|
"text_sha256": "442a5d62ba8c4a3ffda0e97b81c1f2fbdc31da045baa8ce44564e4ce5526f429",
|
|
"full_token_count": 45,
|
|
"token_ids": [
|
|
110708,
|
|
110768,
|
|
96200,
|
|
96144,
|
|
96037,
|
|
97523,
|
|
96127,
|
|
120243,
|
|
3709,
|
|
99442,
|
|
111028,
|
|
5205,
|
|
104619,
|
|
95793,
|
|
112965,
|
|
95726,
|
|
116239,
|
|
1710,
|
|
109080,
|
|
104553,
|
|
113733,
|
|
114462,
|
|
110868,
|
|
3709,
|
|
97787,
|
|
110677,
|
|
95946,
|
|
96218,
|
|
104608,
|
|
95839,
|
|
118309,
|
|
3709
|
|
]
|
|
},
|
|
{
|
|
"id": "cal-en-1",
|
|
"split": "calibration",
|
|
"language": "en",
|
|
"text": "A small library opens every morning beside the river. The librarian checks returned books, repairs damaged covers, and places new arrivals on a wooden table. Visitors often stop to read before continuing their journey across the old bridge.",
|
|
"text_sha256": "c917f3d62e6ecaff19a324e6c0aa1436e33d7d89b436c758458349a4e0e2fe38",
|
|
"full_token_count": 43,
|
|
"token_ids": [
|
|
32,
|
|
2526,
|
|
6522,
|
|
15417,
|
|
1396,
|
|
6353,
|
|
28428,
|
|
279,
|
|
14367,
|
|
13,
|
|
561,
|
|
90956,
|
|
11988,
|
|
5743,
|
|
6265,
|
|
11,
|
|
29201,
|
|
19510,
|
|
14103,
|
|
11,
|
|
321,
|
|
7247,
|
|
491,
|
|
66710,
|
|
383,
|
|
264,
|
|
21664,
|
|
1898,
|
|
13,
|
|
66511,
|
|
3428,
|
|
2842
|
|
]
|
|
},
|
|
{
|
|
"id": "cal-en-2",
|
|
"split": "calibration",
|
|
"language": "en",
|
|
"text": "To compare two measurements, first record their units and the conditions under which they were collected. Compute the difference, inspect unusual observations, and repeat the procedure with independent examples. A short report should separate observations from explanations and unresolved questions.",
|
|
"text_sha256": "ccb9be13ae9a881a9364211b20f31b6e11c75df482a02359625e7021b3869837",
|
|
"full_token_count": 46,
|
|
"token_ids": [
|
|
1206,
|
|
9150,
|
|
1330,
|
|
20934,
|
|
11,
|
|
1118,
|
|
3150,
|
|
836,
|
|
7896,
|
|
321,
|
|
279,
|
|
4530,
|
|
1172,
|
|
864,
|
|
781,
|
|
998,
|
|
14129,
|
|
13,
|
|
21902,
|
|
279,
|
|
6463,
|
|
11,
|
|
23331,
|
|
17957,
|
|
22980,
|
|
11,
|
|
321,
|
|
12771,
|
|
279,
|
|
10018,
|
|
440,
|
|
9209
|
|
]
|
|
},
|
|
{
|
|
"id": "test-zh-1",
|
|
"split": "heldout",
|
|
"language": "zh",
|
|
"text": "\u5348\u540e\uff0c\u5de5\u7a0b\u5e08\u68c0\u67e5\u6e29\u5ba4\u4e2d\u7684\u4f20\u611f\u5668\u3002\u6e29\u5ea6\u8bb0\u5f55\u51fa\u73b0\u4e00\u4e2a\u5f02\u5e38\u503c\uff0c\u5979\u6ca1\u6709\u7acb\u523b\u5220\u9664\uff0c\u800c\u662f\u67e5\u770b\u8bbe\u5907\u72b6\u6001\u5e76\u8865\u505a\u4e00\u6b21\u6d4b\u91cf\u3002\u786e\u8ba4\u539f\u56e0\u4ee5\u540e\uff0c\u5979\u8c03\u6574\u4e86\u91c7\u6837\u8ba1\u5212\uff0c\u5728\u62a5\u544a\u4e2d\u8bf4\u660e\u8fd9\u6b21\u53d8\u5316\u3002",
|
|
"text_sha256": "5f30e958b262b83dd050d5d695b24cd160c966dc4151225c4e4bcb4be1ed7c96",
|
|
"full_token_count": 44,
|
|
"token_ids": [
|
|
118360,
|
|
3709,
|
|
98941,
|
|
97552,
|
|
106098,
|
|
96875,
|
|
108862,
|
|
1710,
|
|
99472,
|
|
98151,
|
|
97150,
|
|
96125,
|
|
100016,
|
|
96326,
|
|
3709,
|
|
133014,
|
|
103924,
|
|
99553,
|
|
3709,
|
|
101594,
|
|
97239,
|
|
96958,
|
|
98822,
|
|
96172,
|
|
96770,
|
|
134943,
|
|
108562,
|
|
1710,
|
|
101632,
|
|
97494,
|
|
99925,
|
|
3709
|
|
]
|
|
},
|
|
{
|
|
"id": "test-zh-2",
|
|
"split": "heldout",
|
|
"language": "zh",
|
|
"text": "\u5468\u672b\uff0c\u5bb6\u4eba\u6cbf\u7740\u6e56\u8fb9\u7684\u5c0f\u8def\u6563\u6b65\u3002\u4ed6\u4eec\u5e26\u4e86\u6c34\u548c\u7b80\u5355\u7684\u5348\u9910\uff0c\u6253\u7b97\u5728\u6811\u6797\u65c1\u4f11\u606f\u3002\u9014\u4e2d\u9047\u5230\u4e00\u5904\u6b63\u5728\u7ef4\u4fee\u7684\u6865\uff0c\u4e8e\u662f\u53c2\u8003\u8def\u6807\u9009\u62e9\u53e6\u4e00\u6761\u9053\u8def\uff0c\u7ec8\u4e8e\u5728\u65e5\u843d\u524d\u56de\u5230\u8f66\u7ad9\u3002",
|
|
"text_sha256": "309dc66b4bb36ee28a0efe3f2ac91bced12fb3727f83fe4f556be583888d2f69",
|
|
"full_token_count": 44,
|
|
"token_ids": [
|
|
104195,
|
|
3709,
|
|
99993,
|
|
110530,
|
|
131667,
|
|
97626,
|
|
96112,
|
|
112869,
|
|
1710,
|
|
96697,
|
|
125485,
|
|
114701,
|
|
101829,
|
|
110637,
|
|
3709,
|
|
108758,
|
|
95772,
|
|
113762,
|
|
97760,
|
|
100020,
|
|
1710,
|
|
104992,
|
|
99475,
|
|
112430,
|
|
99443,
|
|
99523,
|
|
95726,
|
|
97461,
|
|
3709,
|
|
100556,
|
|
97910,
|
|
96112
|
|
]
|
|
},
|
|
{
|
|
"id": "test-en-1",
|
|
"split": "heldout",
|
|
"language": "en",
|
|
"text": "The gardener planted several rows of beans behind the house. Each evening she checked the soil and wrote down how much water the plants received. After a week of warm weather, the first green shoots appeared near the fence beside the apple tree.",
|
|
"text_sha256": "9f25e796901243c848d93ec8c3dd74a13b50e8207388ff41d24cdfba3869db62",
|
|
"full_token_count": 48,
|
|
"token_ids": [
|
|
760,
|
|
21208,
|
|
772,
|
|
37070,
|
|
3679,
|
|
6761,
|
|
314,
|
|
26034,
|
|
4661,
|
|
279,
|
|
3626,
|
|
13,
|
|
8618,
|
|
11129,
|
|
1292,
|
|
9770,
|
|
279,
|
|
16665,
|
|
321,
|
|
5947,
|
|
1441,
|
|
1204,
|
|
1691,
|
|
2919,
|
|
279,
|
|
10462,
|
|
3816,
|
|
13,
|
|
4486,
|
|
264,
|
|
1936,
|
|
314
|
|
]
|
|
},
|
|
{
|
|
"id": "test-en-2",
|
|
"split": "heldout",
|
|
"language": "en",
|
|
"text": "When a program reads a table, it should check column names and handle missing entries explicitly. The developer tested a small example, compared the output with a hand calculation, and saved the input alongside the result so another person could reproduce the check.",
|
|
"text_sha256": "6c47a00b9b81ae364d9cedb79f7ba46fbb104fe90ce35b6e742e45c6b106ae9a",
|
|
"full_token_count": 48,
|
|
"token_ids": [
|
|
4350,
|
|
264,
|
|
1957,
|
|
15339,
|
|
264,
|
|
1898,
|
|
11,
|
|
424,
|
|
1220,
|
|
1716,
|
|
3145,
|
|
4874,
|
|
321,
|
|
3579,
|
|
7167,
|
|
10379,
|
|
20335,
|
|
13,
|
|
561,
|
|
15291,
|
|
12152,
|
|
264,
|
|
2526,
|
|
3010,
|
|
11,
|
|
7463,
|
|
279,
|
|
2468,
|
|
440,
|
|
264,
|
|
1375,
|
|
21261
|
|
]
|
|
}
|
|
]
|
|
}
|