{"files": 2, "bytes": 6814, "documents": 5, "segments": 71, "tokens": 2446, "characters": 4680, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["text", "ts", "cluster_size", "de", "openlid_v2", "seg_langs_openlid_v3", "id", "xml", "crawl_id", "openlid_v3", "glotlid_v3", "filter", "f", "u", "rs", "s", "doc_scores", "pii", "md", "o", "c"], "optional": ["metalang", "htmllang"]}, "name": "ace_Arab", "t/d": 489.2, "c/t": 1.9133278822567457, "samples": "https://data.hplt-project.org/three/sorted/ace_Arab.json", "urls": ["https://data.hplt-project.org/three/sorted/ace_Arab/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ace_Arab/7_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/ace_Arab.map", "md5": "https://data.hplt-project.org/three/sorted/ace_Arab.md5"}
{"files": 6, "bytes": 11652500, "documents": 4910, "segments": 146926, "tokens": 8849227, "characters": 24054858, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["f", "cluster_size", "u", "o", "openlid_v2", "md", "xml", "c", "de", "ts", "filter", "rs", "pii", "text", "id", "glotlid_v3", "s", "doc_scores", "openlid_v3", "seg_langs_openlid_v3", "crawl_id"], "optional": ["allowed", "htmllang", "metalang"]}, "name": "ace_Latn", "t/d": 1802.2865580448065, "c/t": 2.71830047980462, "samples": "https://data.hplt-project.org/three/sorted/ace_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/ace_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ace_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ace_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ace_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ace_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ace_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/ace_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/ace_Latn.md5"}
{"files": 6, "bytes": 5487540576, "documents": 2327934, "segments": 61154956, "tokens": 2964455305, "characters": 9708207795, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["id", "seg_langs_openlid_v3", "md", "cluster_size", "filter", "crawl_id", "o", "ts", "web-register", "pii", "openlid_v2", "c", "openlid_v3", "f", "u", "de", "s", "xml", "text", "rs", "glotlid_v3", "doc_scores"], "optional": ["htmllang", "metalang", "allowed"]}, "name": "afr_Latn", "t/d": 1273.4275563654296, "c/t": 3.274870691632843, "samples": "https://data.hplt-project.org/three/sorted/afr_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/afr_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/afr_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/afr_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/afr_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/afr_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/afr_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/afr_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/afr_Latn.md5"}
{"files": 6, "bytes": 19525213293, "documents": 12179805, "segments": 170458625, "tokens": 10708913885, "characters": 29469731696, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["text", "xml", "md", "f", "o", "s", "rs", "u", "c", "ts", "de", "id", "openlid_v2", "glotlid_v3", "openlid_v3", "seg_langs_openlid_v3", "cluster_size", "web-register", "crawl_id", "filter", "pii", "doc_scores", "bsc-edu", "finepdfs-edu", "jql", "propella-4b"], "optional": ["allowed", "htmllang", "metalang"]}, "name": "als_Latn", "t/d": 879.2352492507064, "c/t": 2.751888007735156, "samples": "https://data.hplt-project.org/three/sorted/als_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/als_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/als_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/als_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/als_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/als_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/als_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/als_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/als_Latn.md5"}
{"files": 6, "bytes": 1798817867, "documents": 626329, "segments": 13384542, "tokens": 1085009054, "characters": 1801401971, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["md", "id", "f", "c", "openlid_v3", "de", "o", "doc_scores", "openlid_v2", "ts", "cluster_size", "u", "text", "crawl_id", "filter", "s", "glotlid_v3", "xml", "rs", "seg_langs_openlid_v3", "web-register", "pii"], "optional": ["allowed", "metalang", "htmllang"]}, "name": "amh_Ethi", "t/d": 1732.3308580634139, "c/t": 1.660264459876111, "samples": "https://data.hplt-project.org/three/sorted/amh_Ethi.json", "urls": ["https://data.hplt-project.org/three/sorted/amh_Ethi/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/amh_Ethi/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/amh_Ethi/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/amh_Ethi/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/amh_Ethi/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/amh_Ethi/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/amh_Ethi.map", "md5": "https://data.hplt-project.org/three/sorted/amh_Ethi.md5"}
{"files": 31, "bytes": 360477609121, "documents": 147106645, "segments": 3319092815, "tokens": 166226814760, "characters": 473482284212, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["crawl_id", "text", "rs", "web-register", "cluster_size", "ts", "f", "seg_langs_openlid_v3", "u", "c", "de", "pii", "id", "filter", "openlid_v2", "md", "openlid_v3", "xml", "doc_scores", "s", "glotlid_v3", "o"], "optional": ["metalang", "allowed", "htmllang"]}, "name": "ara_Arab", "t/d": 1129.9748883539557, "c/t": 2.8484109792732215, "samples": "https://data.hplt-project.org/three/sorted/ara_Arab.json", "urls": ["https://data.hplt-project.org/three/sorted/ara_Arab/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ara_Arab/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ara_Arab/5_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/ara_Arab/5_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/ara_Arab/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ara_Arab/6_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/ara_Arab/6_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/ara_Arab/6_4.jsonl.zst", "https://data.hplt-project.org/three/sorted/ara_Arab/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ara_Arab/7_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/ara_Arab/7_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/ara_Arab/7_4.jsonl.zst", "https://data.hplt-project.org/three/sorted/ara_Arab/7_5.jsonl.zst", "https://data.hplt-project.org/three/sorted/ara_Arab/7_6.jsonl.zst", "https://data.hplt-project.org/three/sorted/ara_Arab/7_7.jsonl.zst", "https://data.hplt-project.org/three/sorted/ara_Arab/7_8.jsonl.zst", "https://data.hplt-project.org/three/sorted/ara_Arab/7_9.jsonl.zst", "https://data.hplt-project.org/three/sorted/ara_Arab/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ara_Arab/8_10.jsonl.zst", "https://data.hplt-project.org/three/sorted/ara_Arab/8_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/ara_Arab/8_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/ara_Arab/8_4.jsonl.zst", "https://data.hplt-project.org/three/sorted/ara_Arab/8_5.jsonl.zst", "https://data.hplt-project.org/three/sorted/ara_Arab/8_6.jsonl.zst", "https://data.hplt-project.org/three/sorted/ara_Arab/8_7.jsonl.zst", "https://data.hplt-project.org/three/sorted/ara_Arab/8_8.jsonl.zst", "https://data.hplt-project.org/three/sorted/ara_Arab/8_9.jsonl.zst", "https://data.hplt-project.org/three/sorted/ara_Arab/9_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ara_Arab/9_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/ara_Arab/9_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/ara_Arab/9_4.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/ara_Arab.map", "md5": "https://data.hplt-project.org/three/sorted/ara_Arab.md5"}
{"files": 6, "bytes": 1159014588, "documents": 494769, "segments": 7225936, "tokens": 544148137, "characters": 1295584461, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["md", "s", "pii", "seg_langs_openlid_v3", "filter", "doc_scores", "c", "cluster_size", "crawl_id", "text", "glotlid_v3", "xml", "u", "openlid_v3", "de", "openlid_v2", "id", "ts", "o", "web-register", "rs", "f"], "optional": ["metalang", "allowed", "htmllang"]}, "name": "asm_Beng", "t/d": 1099.8024067797296, "c/t": 2.3809407271755485, "samples": "https://data.hplt-project.org/three/sorted/asm_Beng.json", "urls": ["https://data.hplt-project.org/three/sorted/asm_Beng/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/asm_Beng/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/asm_Beng/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/asm_Beng/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/asm_Beng/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/asm_Beng/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/asm_Beng.map", "md5": "https://data.hplt-project.org/three/sorted/asm_Beng.md5"}
{"files": 6, "bytes": 426651638, "documents": 175329, "segments": 4122685, "tokens": 230881469, "characters": 758438826, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["glotlid_v3", "u", "pii", "rs", "text", "ts", "o", "cluster_size", "xml", "openlid_v3", "f", "de", "seg_langs_openlid_v3", "doc_scores", "id", "md", "c", "crawl_id", "s", "filter", "openlid_v2"], "optional": ["htmllang", "allowed", "metalang"]}, "name": "ast_Latn", "t/d": 1316.8470076256638, "c/t": 3.2849705491089023, "samples": "https://data.hplt-project.org/three/sorted/ast_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/ast_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ast_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ast_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ast_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ast_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ast_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/ast_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/ast_Latn.md5"}
{"files": 5, "bytes": 14059779, "documents": 5168, "segments": 70594, "tokens": 6289165, "characters": 15717588, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["s", "o", "ts", "u", "filter", "id", "seg_langs_openlid_v3", "rs", "pii", "crawl_id", "c", "cluster_size", "de", "openlid_v3", "glotlid_v3", "md", "f", "text", "xml", "openlid_v2", "doc_scores"], "optional": ["htmllang", "metalang", "allowed"]}, "name": "awa_Deva", "t/d": 1216.9436919504644, "c/t": 2.4991533852268146, "samples": "https://data.hplt-project.org/three/sorted/awa_Deva.json", "urls": ["https://data.hplt-project.org/three/sorted/awa_Deva/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/awa_Deva/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/awa_Deva/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/awa_Deva/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/awa_Deva/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/awa_Deva.map", "md5": "https://data.hplt-project.org/three/sorted/awa_Deva.md5"}
{"files": 6, "bytes": 11088468, "documents": 8636, "segments": 149133, "tokens": 8727740, "characters": 23149306, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["ts", "filter", "xml", "seg_langs_openlid_v3", "doc_scores", "text", "de", "cluster_size", "glotlid_v3", "md", "pii", "openlid_v3", "f", "crawl_id", "u", "id", "rs", "o", "s", "c", "openlid_v2"], "optional": ["htmllang", "allowed", "metalang"]}, "name": "ayr_Latn", "t/d": 1010.6229735988884, "c/t": 2.652382632846533, "samples": "https://data.hplt-project.org/three/sorted/ayr_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/ayr_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ayr_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ayr_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ayr_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ayr_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ayr_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/ayr_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/ayr_Latn.md5"}
{"files": 6, "bytes": 201373858, "documents": 79835, "segments": 2309546, "tokens": 114628036, "characters": 251659699, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["rs", "de", "filter", "id", "md", "ts", "text", "c", "f", "s", "cluster_size", "u", "seg_langs_openlid_v3", "crawl_id", "openlid_v3", "glotlid_v3", "pii", "o", "web-register", "openlid_v2", "doc_scores", "xml"], "optional": ["allowed", "htmllang", "metalang"]}, "name": "azb_Arab", "t/d": 1435.8118118619652, "c/t": 2.195446312977045, "samples": "https://data.hplt-project.org/three/sorted/azb_Arab.json", "urls": ["https://data.hplt-project.org/three/sorted/azb_Arab/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/azb_Arab/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/azb_Arab/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/azb_Arab/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/azb_Arab/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/azb_Arab/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/azb_Arab.map", "md5": "https://data.hplt-project.org/three/sorted/azb_Arab.md5"}
{"files": 6, "bytes": 24137723417, "documents": 12225288, "segments": 266366060, "tokens": 17699762348, "characters": 45883342635, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["crawl_id", "s", "web-register", "text", "cluster_size", "md", "openlid_v2", "de", "id", "f", "c", "doc_scores", "seg_langs_openlid_v3", "glotlid_v3", "openlid_v3", "filter", "o", "ts", "xml", "rs", "u", "pii"], "optional": ["allowed", "metalang", "htmllang"]}, "name": "azj_Latn", "t/d": 1447.7992132373488, "c/t": 2.592314051051913, "samples": "https://data.hplt-project.org/three/sorted/azj_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/azj_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/azj_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/azj_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/azj_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/azj_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/azj_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/azj_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/azj_Latn.md5"}
{"files": 6, "bytes": 550375401, "documents": 282223, "segments": 4134250, "tokens": 394745964, "characters": 814868710, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["openlid_v3", "doc_scores", "text", "u", "md", "seg_langs_openlid_v3", "openlid_v2", "ts", "pii", "o", "de", "crawl_id", "id", "filter", "rs", "glotlid_v3", "cluster_size", "s", "c", "f", "xml"], "optional": ["metalang", "htmllang", "allowed"]}, "name": "bak_Cyrl", "t/d": 1398.7023169621186, "c/t": 2.0642863621526475, "samples": "https://data.hplt-project.org/three/sorted/bak_Cyrl.json", "urls": ["https://data.hplt-project.org/three/sorted/bak_Cyrl/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/bak_Cyrl/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/bak_Cyrl/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/bak_Cyrl/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/bak_Cyrl/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/bak_Cyrl/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/bak_Cyrl.map", "md5": "https://data.hplt-project.org/three/sorted/bak_Cyrl.md5"}
{"files": 5, "bytes": 7427862, "documents": 3978, "segments": 85640, "tokens": 5925867, "characters": 13004832, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["f", "id", "xml", "pii", "o", "cluster_size", "text", "ts", "openlid_v3", "seg_langs_openlid_v3", "rs", "openlid_v2", "crawl_id", "s", "filter", "de", "c", "u", "glotlid_v3", "doc_scores", "md"], "optional": ["allowed", "htmllang", "metalang"]}, "name": "bam_Latn", "t/d": 1489.6598793363498, "c/t": 2.194587222426693, "samples": "https://data.hplt-project.org/three/sorted/bam_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/bam_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/bam_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/bam_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/bam_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/bam_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/bam_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/bam_Latn.md5"}
{"files": 6, "bytes": 23349047, "documents": 10982, "segments": 300780, "tokens": 13683859, "characters": 41555512, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["s", "rs", "doc_scores", "u", "c", "pii", "filter", "text", "xml", "o", "cluster_size", "ts", "seg_langs_openlid_v3", "crawl_id", "id", "glotlid_v3", "md", "de", "openlid_v3", "f", "openlid_v2"], "optional": ["metalang", "htmllang", "allowed"]}, "name": "ban_Latn", "t/d": 1246.0261336732835, "c/t": 3.0368269652588498, "samples": "https://data.hplt-project.org/three/sorted/ban_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/ban_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ban_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ban_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ban_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ban_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ban_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/ban_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/ban_Latn.md5"}
{"files": 6, "bytes": 8463061385, "documents": 3431734, "segments": 61832138, "tokens": 4510902466, "characters": 11238867097, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["de", "o", "pii", "crawl_id", "rs", "s", "openlid_v3", "seg_langs_openlid_v3", "c", "cluster_size", "filter", "xml", "glotlid_v3", "md", "text", "doc_scores", "u", "ts", "web-register", "id", "openlid_v2", "f"], "optional": ["allowed", "metalang", "htmllang"]}, "name": "bel_Cyrl", "t/d": 1314.467399279781, "c/t": 2.4914897144663737, "samples": "https://data.hplt-project.org/three/sorted/bel_Cyrl.json", "urls": ["https://data.hplt-project.org/three/sorted/bel_Cyrl/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/bel_Cyrl/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/bel_Cyrl/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/bel_Cyrl/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/bel_Cyrl/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/bel_Cyrl/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/bel_Cyrl.map", "md5": "https://data.hplt-project.org/three/sorted/bel_Cyrl.md5"}
{"files": 6, "bytes": 7364880, "documents": 1124, "segments": 44677, "tokens": 6496692, "characters": 17782328, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["crawl_id", "text", "de", "c", "md", "s", "seg_langs_openlid_v3", "id", "u", "cluster_size", "ts", "xml", "o", "glotlid_v3", "doc_scores", "filter", "openlid_v2", "rs", "f", "pii", "openlid_v3"], "optional": ["allowed", "metalang", "htmllang"]}, "name": "bem_Latn", "t/d": 5779.975088967972, "c/t": 2.7371357607841036, "samples": "https://data.hplt-project.org/three/sorted/bem_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/bem_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/bem_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/bem_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/bem_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/bem_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/bem_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/bem_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/bem_Latn.md5"}
{"files": 10, "bytes": 62167804644, "documents": 29558506, "segments": 397260027, "tokens": 18356780420, "characters": 70465978145, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["pii", "rs", "f", "de", "u", "o", "s", "doc_scores", "cluster_size", "filter", "seg_langs_openlid_v3", "c", "openlid_v3", "xml", "id", "openlid_v2", "text", "glotlid_v3", "ts", "md", "web-register", "crawl_id"], "optional": ["allowed", "metalang", "htmllang"]}, "name": "ben_Beng", "t/d": 621.0320785495721, "c/t": 3.8386893852162776, "samples": "https://data.hplt-project.org/three/sorted/ben_Beng.json", "urls": ["https://data.hplt-project.org/three/sorted/ben_Beng/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ben_Beng/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ben_Beng/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ben_Beng/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ben_Beng/7_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/ben_Beng/7_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/ben_Beng/7_4.jsonl.zst", "https://data.hplt-project.org/three/sorted/ben_Beng/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ben_Beng/8_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/ben_Beng/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/ben_Beng.map", "md5": "https://data.hplt-project.org/three/sorted/ben_Beng.md5"}
{"files": 6, "bytes": 61240567, "documents": 30326, "segments": 439656, "tokens": 25378433, "characters": 76355313, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["f", "cluster_size", "c", "pii", "ts", "rs", "s", "filter", "o", "openlid_v2", "openlid_v3", "u", "id", "seg_langs_openlid_v3", "doc_scores", "text", "md", "xml", "de", "glotlid_v3", "crawl_id"], "optional": ["allowed", "htmllang", "metalang"]}, "name": "bho_Deva", "t/d": 836.8539537030931, "c/t": 3.008669329583903, "samples": "https://data.hplt-project.org/three/sorted/bho_Deva.json", "urls": ["https://data.hplt-project.org/three/sorted/bho_Deva/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/bho_Deva/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/bho_Deva/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/bho_Deva/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/bho_Deva/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/bho_Deva/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/bho_Deva.map", "md5": "https://data.hplt-project.org/three/sorted/bho_Deva.md5"}
{"files": 5, "bytes": 2558908, "documents": 1200, "segments": 23981, "tokens": 1780598, "characters": 3569187, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["crawl_id", "cluster_size", "c", "seg_langs_openlid_v3", "o", "openlid_v3", "id", "u", "f", "filter", "glotlid_v3", "s", "text", "xml", "doc_scores", "rs", "pii", "de", "openlid_v2", "md", "ts"], "optional": ["allowed", "metalang", "htmllang"]}, "name": "bjn_Arab", "t/d": 1483.8316666666667, "c/t": 2.0044878181375023, "samples": "https://data.hplt-project.org/three/sorted/bjn_Arab.json", "urls": ["https://data.hplt-project.org/three/sorted/bjn_Arab/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/bjn_Arab/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/bjn_Arab/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/bjn_Arab/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/bjn_Arab/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/bjn_Arab.map", "md5": "https://data.hplt-project.org/three/sorted/bjn_Arab.md5"}
{"files": 6, "bytes": 10861871, "documents": 5578, "segments": 136834, "tokens": 6169978, "characters": 19979977, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["c", "doc_scores", "ts", "seg_langs_openlid_v3", "id", "filter", "pii", "crawl_id", "u", "s", "glotlid_v3", "openlid_v3", "md", "xml", "text", "rs", "de", "o", "cluster_size", "f", "openlid_v2"], "optional": ["metalang", "htmllang", "allowed"]}, "name": "bjn_Latn", "t/d": 1106.1272857655074, "c/t": 3.2382574135596593, "samples": "https://data.hplt-project.org/three/sorted/bjn_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/bjn_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/bjn_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/bjn_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/bjn_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/bjn_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/bjn_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/bjn_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/bjn_Latn.md5"}
{"files": 6, "bytes": 13773922, "documents": 3723, "segments": 60257, "tokens": 11808759, "characters": 19066750, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["seg_langs_openlid_v3", "openlid_v2", "text", "filter", "openlid_v3", "crawl_id", "id", "cluster_size", "doc_scores", "xml", "de", "u", "glotlid_v3", "pii", "rs", "o", "c", "f", "ts", "s", "md"], "optional": ["metalang", "allowed", "htmllang"]}, "name": "bod_Tibt", "t/d": 3171.83964544722, "c/t": 1.614627752162611, "samples": "https://data.hplt-project.org/three/sorted/bod_Tibt.json", "urls": ["https://data.hplt-project.org/three/sorted/bod_Tibt/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/bod_Tibt/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/bod_Tibt/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/bod_Tibt/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/bod_Tibt/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/bod_Tibt/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/bod_Tibt.map", "md5": "https://data.hplt-project.org/three/sorted/bod_Tibt.md5"}
{"files": 6, "bytes": 17044757298, "documents": 10323906, "segments": 132038529, "tokens": 8471053599, "characters": 26693525767, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["text", "xml", "md", "f", "o", "s", "rs", "u", "c", "ts", "de", "id", "openlid_v2", "glotlid_v3", "openlid_v3", "seg_langs_openlid_v3", "cluster_size", "crawl_id", "filter", "pii", "doc_scores", "bsc-edu", "finepdfs-edu", "jql", "propella-4b"], "optional": ["allowed", "htmllang", "metalang"]}, "name": "bos_Latn", "t/d": 820.5279667404953, "c/t": 3.151145894077585, "samples": "https://data.hplt-project.org/three/sorted/bos_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/bos_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/bos_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/bos_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/bos_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/bos_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/bos_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/bos_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/bos_Latn.md5"}
{"files": 6, "bytes": 2424960, "documents": 780, "segments": 28357, "tokens": 2007121, "characters": 5408755, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["u", "crawl_id", "s", "o", "md", "seg_langs_openlid_v3", "openlid_v3", "rs", "doc_scores", "f", "ts", "id", "cluster_size", "glotlid_v3", "xml", "text", "c", "pii", "filter", "de", "openlid_v2"], "optional": ["allowed", "metalang", "htmllang"]}, "name": "bug_Latn", "t/d": 2573.232051282051, "c/t": 2.694782726103708, "samples": "https://data.hplt-project.org/three/sorted/bug_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/bug_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/bug_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/bug_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/bug_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/bug_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/bug_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/bug_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/bug_Latn.md5"}
{"files": 13, "bytes": 125476117335, "documents": 46593078, "segments": 1072016517, "tokens": 54063606791, "characters": 160701425963, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["text", "xml", "md", "f", "o", "s", "rs", "u", "c", "ts", "de", "id", "openlid_v2", "glotlid_v3", "openlid_v3", "seg_langs_openlid_v3", "cluster_size", "web-register", "crawl_id", "filter", "pii", "doc_scores", "bsc-edu", "finepdfs-edu", "jql", "propella-4b"], "optional": ["htmllang", "metalang", "allowed"]}, "name": "bul_Cyrl", "t/d": 1160.3355930037505, "c/t": 2.9724510720167503, "samples": "https://data.hplt-project.org/three/sorted/bul_Cyrl.json", "urls": ["https://data.hplt-project.org/three/sorted/bul_Cyrl/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/bul_Cyrl/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/bul_Cyrl/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/bul_Cyrl/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/bul_Cyrl/7_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/bul_Cyrl/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/bul_Cyrl/8_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/bul_Cyrl/8_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/bul_Cyrl/8_4.jsonl.zst", "https://data.hplt-project.org/three/sorted/bul_Cyrl/9_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/bul_Cyrl/9_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/bul_Cyrl/9_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/bul_Cyrl/9_4.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/bul_Cyrl.map", "md5": "https://data.hplt-project.org/three/sorted/bul_Cyrl.md5"}
{"files": 7, "bytes": 50279331172, "documents": 27614886, "segments": 461080365, "tokens": 22786948572, "characters": 76249509615, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["text", "xml", "md", "f", "o", "s", "rs", "u", "c", "ts", "de", "id", "openlid_v2", "glotlid_v3", "openlid_v3", "seg_langs_openlid_v3", "cluster_size", "web-register", "crawl_id", "filter", "pii", "doc_scores", "bsc-edu", "finepdfs-edu", "jql", "propella-4b"], "optional": ["htmllang", "metalang", "allowed"]}, "name": "cat_Latn", "t/d": 825.1690255755537, "c/t": 3.346192201824398, "samples": "https://data.hplt-project.org/three/sorted/cat_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/cat_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/cat_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/cat_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/cat_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/cat_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/cat_Latn/8_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/cat_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/cat_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/cat_Latn.md5"}
{"files": 6, "bytes": 623340605, "documents": 355602, "segments": 6891703, "tokens": 383812969, "characters": 1263029638, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["c", "de", "u", "id", "text", "glotlid_v3", "xml", "o", "pii", "crawl_id", "s", "doc_scores", "openlid_v2", "md", "ts", "seg_langs_openlid_v3", "rs", "openlid_v3", "cluster_size", "f", "filter"], "optional": ["metalang", "allowed", "htmllang"]}, "name": "ceb_Latn", "t/d": 1079.3329874410156, "c/t": 3.2907424709767947, "samples": "https://data.hplt-project.org/three/sorted/ceb_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/ceb_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ceb_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ceb_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ceb_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ceb_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ceb_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/ceb_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/ceb_Latn.md5"}
{"files": 19, "bytes": 270263381212, "documents": 117324028, "segments": 2574322034, "tokens": 131905209533, "characters": 384235238644, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["text", "xml", "md", "f", "o", "s", "rs", "u", "c", "ts", "de", "id", "openlid_v2", "glotlid_v3", "openlid_v3", "seg_langs_openlid_v3", "cluster_size", "web-register", "crawl_id", "filter", "pii", "doc_scores", "bsc-edu", "finepdfs-edu", "fineweb2-hq", "jql", "propella-4b"], "optional": ["allowed", "htmllang", "metalang"]}, "name": "ces_Latn", "t/d": 1124.2812898735458, "c/t": 2.9129648480477353, "samples": "https://data.hplt-project.org/three/sorted/ces_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/ces_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ces_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ces_Latn/5_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/ces_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ces_Latn/6_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/ces_Latn/6_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/ces_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ces_Latn/7_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/ces_Latn/7_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/ces_Latn/7_4.jsonl.zst", "https://data.hplt-project.org/three/sorted/ces_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ces_Latn/8_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/ces_Latn/8_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/ces_Latn/8_4.jsonl.zst", "https://data.hplt-project.org/three/sorted/ces_Latn/8_5.jsonl.zst", "https://data.hplt-project.org/three/sorted/ces_Latn/9_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ces_Latn/9_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/ces_Latn/9_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/ces_Latn/9_4.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/ces_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/ces_Latn.md5"}
{"files": 5, "bytes": 1203734, "documents": 477, "segments": 10693, "tokens": 1010000, "characters": 2673319, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["seg_langs_openlid_v3", "text", "u", "doc_scores", "c", "glotlid_v3", "ts", "f", "xml", "id", "s", "crawl_id", "pii", "openlid_v2", "cluster_size", "rs", "o", "de", "openlid_v3", "filter", "md"], "optional": ["allowed", "metalang", "htmllang"]}, "name": "cjk_Latn", "t/d": 2117.4004192872117, "c/t": 2.646850495049505, "samples": "https://data.hplt-project.org/three/sorted/cjk_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/cjk_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/cjk_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/cjk_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/cjk_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/cjk_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/cjk_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/cjk_Latn.md5"}
{"files": 6, "bytes": 658624630, "documents": 341097, "segments": 4751368, "tokens": 446382326, "characters": 909635959, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["openlid_v3", "filter", "o", "xml", "text", "glotlid_v3", "u", "rs", "ts", "seg_langs_openlid_v3", "s", "web-register", "crawl_id", "pii", "id", "openlid_v2", "md", "cluster_size", "doc_scores", "de", "c", "f"], "optional": ["allowed", "metalang", "htmllang"]}, "name": "ckb_Arab", "t/d": 1308.66681911597, "c/t": 2.037795642921579, "samples": "https://data.hplt-project.org/three/sorted/ckb_Arab.json", "urls": ["https://data.hplt-project.org/three/sorted/ckb_Arab/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ckb_Arab/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ckb_Arab/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ckb_Arab/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ckb_Arab/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ckb_Arab/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/ckb_Arab.map", "md5": "https://data.hplt-project.org/three/sorted/ckb_Arab.md5"}
{"files": 304, "bytes": 5257460999187, "documents": 1735837599, "segments": 47142954769, "tokens": 2277409013445, "characters": 3171182312567, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["text", "xml", "md", "f", "o", "s", "rs", "u", "c", "ts", "de", "id", "openlid_v2", "allowed", "glotlid_v3", "openlid_v3", "seg_langs_openlid_v3", "cluster_size", "web-register", "crawl_id", "filter", "pii", "doc_scores"], "optional": ["htmllang", "metalang"]}, "name": "cmn_Hans", "t/d": 1311.994287228825, "c/t": 1.3924518142527256, "samples": "https://data.hplt-project.org/three/sorted/cmn_Hans.json", "urls": ["https://data.hplt-project.org/three/sorted/cmn_Hans/10_1.ab.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/10_1.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/10_1.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/10_2.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/10_2.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/10_3.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/5_1.ab.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/5_1.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/5_1.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/5_2.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/5_2.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/5_3.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/5_3.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/5_4.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/5_4.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/5_5.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/5_5.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/5_6.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/5_6.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/5_7.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/5_7.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/6_1.ab.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/6_1.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/6_1.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/6_10.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/6_11.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/6_2.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/6_2.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/6_3.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/6_3.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/6_4.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/6_4.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/6_5.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/6_5.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/6_6.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/6_6.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/6_7.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/6_7.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/6_8.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/6_8.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/6_9.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/6_9.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/7_1.ab.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/7_1.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/7_1.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/7_10.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/7_10.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/7_11.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/7_11.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/7_12.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/7_12.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/7_13.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/7_13.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/7_14.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/7_14.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/7_15.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/7_15.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/7_16.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/7_16.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/7_17.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/7_17.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/7_18.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/7_18.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/7_19.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/7_19.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/7_2.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/7_2.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/7_20.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/7_21.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/7_3.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/7_3.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/7_4.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/7_4.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/7_5.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/7_5.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/7_6.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/7_6.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/7_7.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/7_7.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/7_8.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/7_8.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/7_9.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/7_9.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/8_1.ab.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/8_1.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/8_1.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/8_10.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/8_10.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/8_11.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/8_11.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/8_12.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/8_12.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/8_13.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/8_13.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/8_14.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/8_14.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/8_15.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/8_15.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/8_16.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/8_16.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/8_17.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/8_17.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/8_18.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/8_18.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/8_19.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/8_19.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/8_2.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/8_2.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/8_20.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/8_20.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/8_21.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/8_21.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/8_22.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/8_22.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/8_23.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/8_23.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/8_24.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/8_24.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/8_25.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/8_25.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/8_26.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/8_26.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/8_27.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/8_27.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/8_28.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/8_28.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/8_29.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/8_29.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/8_3.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/8_3.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/8_30.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/8_30.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/8_31.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/8_31.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/8_32.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/8_32.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/8_33.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/8_33.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/8_34.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/8_34.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/8_35.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/8_35.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/8_36.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/8_36.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/8_37.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/8_37.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/8_38.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/8_38.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/8_39.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/8_39.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/8_4.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/8_4.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/8_40.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/8_40.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/8_41.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/8_41.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/8_42.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/8_42.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/8_43.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/8_44.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/8_45.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/8_46.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/8_5.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/8_5.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/8_6.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/8_6.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/8_7.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/8_7.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/8_8.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/8_8.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/8_9.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/8_9.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_1.ab.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_1.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_1.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_10.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_10.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_11.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_11.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_12.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_12.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_13.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_13.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_14.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_14.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_15.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_15.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_16.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_16.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_17.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_17.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_18.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_18.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_19.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_19.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_2.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_2.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_20.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_20.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_21.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_21.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_22.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_22.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_23.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_23.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_24.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_24.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_25.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_25.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_26.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_26.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_27.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_27.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_28.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_28.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_29.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_29.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_3.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_3.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_30.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_30.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_31.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_31.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_32.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_32.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_33.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_33.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_34.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_34.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_35.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_35.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_36.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_36.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_37.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_37.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_38.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_38.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_39.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_39.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_4.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_4.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_40.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_40.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_41.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_41.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_42.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_42.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_43.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_43.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_44.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_44.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_45.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_45.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_46.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_46.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_47.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_47.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_48.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_48.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_49.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_49.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_5.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_5.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_50.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_50.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_51.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_51.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_52.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_52.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_53.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_53.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_54.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_54.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_55.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_55.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_56.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_56.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_57.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_57.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_58.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_58.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_59.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_59.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_6.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_6.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_60.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_61.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_62.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_63.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_64.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_65.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_66.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_67.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_68.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_69.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_7.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_7.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_70.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_71.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_72.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_8.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_8.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_9.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hans/9_9.ia.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/cmn_Hans.map", "md5": "https://data.hplt-project.org/three/sorted/cmn_Hans.md5"}
{"files": 29, "bytes": 401775445080, "documents": 166993651, "segments": 3784792938, "tokens": 208291918373, "characters": 275742479512, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["cluster_size", "o", "openlid_v2", "filter", "xml", "pii", "doc_scores", "glotlid_v3", "c", "openlid_v3", "s", "u", "rs", "web-register", "f", "id", "md", "seg_langs_openlid_v3", "de", "ts", "text", "crawl_id"], "optional": ["allowed", "metalang", "htmllang"]}, "name": "cmn_Hant", "t/d": 1247.3044162199915, "c/t": 1.323827067635973, "samples": "https://data.hplt-project.org/three/sorted/cmn_Hant.json", "urls": ["https://data.hplt-project.org/three/sorted/cmn_Hant/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hant/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hant/5_10.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hant/5_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hant/5_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hant/5_4.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hant/5_5.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hant/5_6.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hant/5_7.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hant/5_8.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hant/5_9.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hant/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hant/6_10.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hant/6_11.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hant/6_12.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hant/6_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hant/6_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hant/6_4.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hant/6_5.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hant/6_6.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hant/6_7.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hant/6_8.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hant/6_9.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hant/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hant/7_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hant/7_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hant/7_4.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hant/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/cmn_Hant/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/cmn_Hant.map", "md5": "https://data.hplt-project.org/three/sorted/cmn_Hant.md5"}
{"files": 6, "bytes": 109300784, "documents": 73542, "segments": 996618, "tokens": 77489831, "characters": 198433333, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["u", "id", "xml", "de", "openlid_v3", "s", "rs", "seg_langs_openlid_v3", "filter", "crawl_id", "text", "o", "f", "glotlid_v3", "cluster_size", "md", "ts", "c", "pii", "openlid_v2", "doc_scores"], "optional": ["metalang", "htmllang", "allowed"]}, "name": "crh_Latn", "t/d": 1053.6813113594953, "c/t": 2.5607661087814217, "samples": "https://data.hplt-project.org/three/sorted/crh_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/crh_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/crh_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/crh_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/crh_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/crh_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/crh_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/crh_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/crh_Latn.md5"}
{"files": 6, "bytes": 2641265361, "documents": 1310828, "segments": 28775183, "tokens": 1816995737, "characters": 4789033867, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["xml", "text", "glotlid_v3", "seg_langs_openlid_v3", "crawl_id", "md", "cluster_size", "o", "pii", "u", "de", "s", "c", "ts", "filter", "doc_scores", "rs", "f", "openlid_v2", "id", "openlid_v3", "web-register"], "optional": ["htmllang", "metalang", "allowed"]}, "name": "cym_Latn", "t/d": 1386.1435192107583, "c/t": 2.63568800381836, "samples": "https://data.hplt-project.org/three/sorted/cym_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/cym_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/cym_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/cym_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/cym_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/cym_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/cym_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/cym_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/cym_Latn.md5"}
{"files": 10, "bytes": 121397380182, "documents": 57754683, "segments": 1273648986, "tokens": 60702044338, "characters": 201592747597, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["text", "xml", "md", "f", "o", "s", "rs", "u", "c", "ts", "de", "id", "openlid_v2", "glotlid_v3", "openlid_v3", "seg_langs_openlid_v3", "cluster_size", "web-register", "crawl_id", "filter", "pii", "doc_scores", "bsc-edu", "finepdfs-edu", "fineweb2-hq", "jql", "propella-4b"], "optional": ["allowed", "htmllang", "metalang"]}, "name": "dan_Latn", "t/d": 1051.0324216998993, "c/t": 3.3210207299526027, "samples": "https://data.hplt-project.org/three/sorted/dan_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/dan_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/dan_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/dan_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/dan_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/dan_Latn/7_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/dan_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/dan_Latn/8_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/dan_Latn/8_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/dan_Latn/9_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/dan_Latn/9_2.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/dan_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/dan_Latn.md5"}
{"files": 156, "bytes": 2521628725533, "documents": 1054346685, "segments": 25390326107, "tokens": 1037799680500, "characters": 4127639422125, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["md", "openlid_v2", "propella-4b", "o", "de", "fineweb2-hq", "pii", "xml", "f", "crawl_id", "filter", "seg_langs_openlid_v3", "glotlid_v3", "rs", "id", "s", "openlid_v3", "finepdfs-edu", "jql", "ts", "c", "bsc-edu", "u", "web-register", "cluster_size", "text", "doc_scores"], "optional": ["metalang", "htmllang", "allowed"]}, "name": "deu_Latn", "t/d": 984.305916891084, "c/t": 3.9772987982963635, "samples": "https://data.hplt-project.org/three/sorted/deu_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/deu_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/10_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/5_10.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/5_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/5_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/5_4.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/5_5.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/5_6.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/5_7.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/5_8.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/5_9.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/6_10.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/6_11.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/6_12.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/6_13.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/6_14.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/6_15.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/6_16.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/6_17.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/6_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/6_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/6_4.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/6_5.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/6_6.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/6_7.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/6_8.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/6_9.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/7_10.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/7_11.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/7_12.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/7_13.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/7_14.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/7_15.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/7_16.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/7_17.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/7_18.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/7_19.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/7_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/7_20.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/7_21.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/7_22.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/7_23.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/7_24.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/7_25.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/7_26.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/7_27.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/7_28.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/7_29.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/7_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/7_30.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/7_31.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/7_32.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/7_4.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/7_5.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/7_6.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/7_7.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/7_8.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/7_9.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/8_10.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/8_11.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/8_12.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/8_13.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/8_14.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/8_15.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/8_16.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/8_17.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/8_18.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/8_19.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/8_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/8_20.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/8_21.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/8_22.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/8_23.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/8_24.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/8_25.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/8_26.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/8_27.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/8_28.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/8_29.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/8_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/8_30.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/8_31.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/8_32.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/8_33.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/8_34.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/8_35.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/8_36.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/8_37.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/8_38.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/8_39.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/8_4.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/8_40.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/8_41.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/8_42.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/8_43.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/8_44.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/8_45.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/8_46.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/8_47.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/8_48.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/8_49.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/8_5.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/8_50.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/8_6.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/8_7.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/8_8.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/8_9.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/9_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/9_10.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/9_11.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/9_12.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/9_13.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/9_14.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/9_15.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/9_16.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/9_17.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/9_18.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/9_19.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/9_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/9_20.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/9_21.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/9_22.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/9_23.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/9_24.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/9_25.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/9_26.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/9_27.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/9_28.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/9_29.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/9_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/9_30.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/9_31.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/9_32.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/9_33.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/9_34.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/9_35.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/9_36.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/9_37.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/9_38.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/9_39.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/9_4.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/9_40.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/9_41.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/9_42.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/9_43.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/9_44.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/9_45.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/9_5.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/9_6.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/9_7.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/9_8.jsonl.zst", "https://data.hplt-project.org/three/sorted/deu_Latn/9_9.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/deu_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/deu_Latn.md5"}
{"files": 5, "bytes": 2711296, "documents": 940, "segments": 33276, "tokens": 2837792, "characters": 5603171, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["ts", "s", "cluster_size", "de", "openlid_v3", "xml", "pii", "text", "u", "filter", "rs", "id", "openlid_v2", "c", "glotlid_v3", "seg_langs_openlid_v3", "doc_scores", "crawl_id", "md", "f", "o"], "optional": ["metalang", "htmllang", "allowed"]}, "name": "dik_Latn", "t/d": 3018.927659574468, "c/t": 1.9744826259288912, "samples": "https://data.hplt-project.org/three/sorted/dik_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/dik_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/dik_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/dik_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/dik_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/dik_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/dik_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/dik_Latn.md5"}
{"files": 1, "bytes": 1057, "documents": 1, "segments": 1, "tokens": 232, "characters": 534, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["crawl_id", "ts", "filter", "de", "o", "pii", "cluster_size", "md", "htmllang", "f", "openlid_v3", "u", "c", "doc_scores", "s", "id", "openlid_v2", "xml", "rs", "seg_langs_openlid_v3", "text", "glotlid_v3"], "optional": []}, "name": "dzo_Tibt", "t/d": 232.0, "c/t": 2.3017241379310347, "samples": "https://data.hplt-project.org/three/sorted/dzo_Tibt.json", "urls": ["https://data.hplt-project.org/three/sorted/dzo_Tibt/7_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/dzo_Tibt.map", "md5": "https://data.hplt-project.org/three/sorted/dzo_Tibt.md5"}
{"files": 6, "bytes": 37541120839, "documents": 15024995, "segments": 426776899, "tokens": 20672195227, "characters": 60811817489, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["text", "xml", "md", "f", "o", "s", "rs", "u", "c", "ts", "de", "id", "openlid_v2", "glotlid_v3", "openlid_v3", "seg_langs_openlid_v3", "cluster_size", "web-register", "crawl_id", "filter", "pii", "doc_scores", "bsc-edu", "finepdfs-edu", "jql", "propella-4b"], "optional": ["allowed", "htmllang", "metalang"]}, "name": "ekk_Latn", "t/d": 1375.8537175553136, "c/t": 2.9417203553483064, "samples": "https://data.hplt-project.org/three/sorted/ekk_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/ekk_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ekk_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ekk_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ekk_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ekk_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ekk_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/ekk_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/ekk_Latn.md5"}
{"files": 21, "bytes": 228762532266, "documents": 91479404, "segments": 1823550107, "tokens": 114872166128, "characters": 290827792459, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["text", "xml", "md", "f", "o", "s", "rs", "u", "c", "ts", "de", "id", "openlid_v2", "glotlid_v3", "openlid_v3", "seg_langs_openlid_v3", "cluster_size", "web-register", "crawl_id", "filter", "pii", "doc_scores", "bsc-edu", "finepdfs-edu", "fineweb2-hq", "jql", "propella-4b"], "optional": ["allowed", "htmllang", "metalang"]}, "name": "ell_Grek", "t/d": 1255.7161623833929, "c/t": 2.531751618011066, "samples": "https://data.hplt-project.org/three/sorted/ell_Grek.json", "urls": ["https://data.hplt-project.org/three/sorted/ell_Grek/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ell_Grek/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ell_Grek/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ell_Grek/6_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/ell_Grek/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ell_Grek/7_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/ell_Grek/7_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/ell_Grek/7_4.jsonl.zst", "https://data.hplt-project.org/three/sorted/ell_Grek/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ell_Grek/8_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/ell_Grek/8_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/ell_Grek/8_4.jsonl.zst", "https://data.hplt-project.org/three/sorted/ell_Grek/8_5.jsonl.zst", "https://data.hplt-project.org/three/sorted/ell_Grek/8_6.jsonl.zst", "https://data.hplt-project.org/three/sorted/ell_Grek/8_7.jsonl.zst", "https://data.hplt-project.org/three/sorted/ell_Grek/9_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ell_Grek/9_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/ell_Grek/9_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/ell_Grek/9_4.jsonl.zst", "https://data.hplt-project.org/three/sorted/ell_Grek/9_5.jsonl.zst", "https://data.hplt-project.org/three/sorted/ell_Grek/9_6.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/ell_Grek.map", "md5": "https://data.hplt-project.org/three/sorted/ell_Grek.md5"}
{"files": 1108, "bytes": 18243341562942, "documents": 7863621111, "segments": 198738017399, "tokens": 7236738433280, "characters": 32132228067869, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["id", "f", "ts", "web-register", "pii", "doc_scores", "c", "bsc-edu", "crawl_id", "u", "de", "filter", "o", "s", "text", "rs", "openlid_v2", "cluster_size", "openlid_v3", "md", "glotlid_v3", "seg_langs_openlid_v3", "xml"], "optional": ["allowed", "metalang", "htmllang"]}, "name": "eng_Latn", "t/d": 920.28066092311, "c/t": 4.44015330443625, "samples": "https://data.hplt-project.org/three/sorted/eng_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/eng_Latn/10_1.ab.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/10_1.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/10_1.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/10_2.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/10_3.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/10_4.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/10_5.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/10_6.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/5_1.ab.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/5_1.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/5_1.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/5_10.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/5_11.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/5_12.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/5_13.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/5_14.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/5_15.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/5_16.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/5_17.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/5_18.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/5_19.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/5_2.ab.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/5_2.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/5_2.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/5_20.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/5_21.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/5_22.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/5_23.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/5_24.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/5_25.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/5_26.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/5_27.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/5_28.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/5_29.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/5_3.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/5_3.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/5_30.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/5_31.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/5_32.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/5_33.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/5_34.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/5_35.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/5_36.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/5_37.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/5_38.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/5_39.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/5_4.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/5_4.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/5_40.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/5_41.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/5_42.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/5_43.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/5_44.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/5_45.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/5_46.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/5_47.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/5_48.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/5_49.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/5_5.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/5_5.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/5_50.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/5_51.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/5_6.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/5_6.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/5_7.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/5_7.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/5_8.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/5_8.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/5_9.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/5_9.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_1.ab.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_1.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_1.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_10.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_10.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_11.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_11.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_12.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_12.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_13.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_13.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_14.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_15.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_16.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_17.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_18.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_19.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_2.ab.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_2.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_2.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_20.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_21.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_22.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_23.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_24.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_25.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_26.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_27.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_28.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_29.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_3.ab.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_3.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_3.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_30.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_31.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_32.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_33.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_34.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_35.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_36.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_37.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_38.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_39.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_4.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_4.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_40.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_41.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_42.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_43.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_44.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_45.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_46.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_47.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_48.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_49.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_5.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_5.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_50.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_51.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_52.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_53.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_54.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_55.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_56.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_57.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_58.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_59.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_6.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_6.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_60.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_61.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_62.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_63.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_64.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_65.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_66.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_67.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_68.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_69.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_7.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_7.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_70.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_71.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_72.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_73.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_74.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_75.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_76.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_77.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_78.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_79.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_8.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_8.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_80.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_81.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_82.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_83.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_84.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_85.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_86.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_87.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_88.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_89.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_9.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_9.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_90.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_91.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/6_92.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_1.ab.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_1.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_1.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_10.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_10.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_100.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_101.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_102.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_103.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_104.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_105.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_106.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_107.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_108.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_109.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_11.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_11.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_110.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_111.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_112.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_113.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_114.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_115.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_116.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_117.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_118.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_119.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_12.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_12.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_120.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_121.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_122.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_123.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_124.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_125.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_126.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_127.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_128.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_129.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_13.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_13.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_130.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_131.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_132.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_133.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_134.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_135.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_136.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_137.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_138.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_139.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_14.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_14.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_140.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_141.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_142.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_143.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_144.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_145.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_146.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_147.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_148.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_149.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_15.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_15.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_150.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_151.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_152.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_153.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_154.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_155.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_156.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_157.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_158.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_159.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_16.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_16.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_160.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_161.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_162.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_163.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_164.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_165.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_166.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_167.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_168.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_169.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_17.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_17.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_170.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_171.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_172.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_173.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_174.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_175.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_176.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_177.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_178.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_179.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_18.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_18.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_180.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_181.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_182.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_183.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_184.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_185.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_186.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_187.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_188.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_189.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_19.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_19.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_190.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_191.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_192.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_193.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_194.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_195.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_196.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_197.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_198.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_199.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_2.ab.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_2.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_2.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_20.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_20.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_200.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_201.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_202.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_203.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_204.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_205.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_206.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_207.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_208.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_209.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_21.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_21.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_210.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_211.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_22.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_22.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_23.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_23.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_24.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_24.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_25.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_26.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_27.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_28.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_29.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_3.ab.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_3.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_3.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_30.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_31.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_32.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_33.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_34.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_35.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_36.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_37.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_38.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_39.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_4.ab.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_4.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_4.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_40.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_41.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_42.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_43.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_44.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_45.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_46.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_47.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_48.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_49.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_5.ab.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_5.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_5.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_50.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_51.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_52.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_53.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_54.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_55.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_56.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_57.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_58.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_59.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_6.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_6.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_60.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_61.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_62.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_63.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_64.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_65.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_66.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_67.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_68.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_69.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_7.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_7.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_70.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_71.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_72.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_73.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_74.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_75.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_76.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_77.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_78.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_79.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_8.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_8.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_80.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_81.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_82.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_83.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_84.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_85.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_86.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_87.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_88.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_89.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_9.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_9.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_90.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_91.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_92.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_93.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_94.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_95.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_96.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_97.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_98.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/7_99.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_1.ab.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_1.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_1.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_10.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_10.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_100.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_101.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_102.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_103.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_104.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_105.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_106.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_107.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_108.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_109.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_11.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_11.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_110.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_111.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_112.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_113.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_114.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_115.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_116.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_117.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_118.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_119.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_12.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_12.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_120.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_121.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_122.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_123.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_124.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_125.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_126.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_127.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_128.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_129.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_13.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_13.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_130.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_131.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_132.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_133.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_134.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_135.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_136.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_137.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_138.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_139.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_14.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_14.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_140.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_141.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_142.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_143.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_144.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_145.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_146.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_147.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_148.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_149.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_15.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_15.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_150.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_151.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_152.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_153.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_154.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_155.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_156.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_157.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_158.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_159.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_16.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_16.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_160.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_161.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_162.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_163.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_164.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_165.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_166.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_167.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_168.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_169.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_17.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_17.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_170.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_171.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_172.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_173.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_174.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_175.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_176.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_177.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_178.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_179.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_18.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_18.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_180.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_181.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_182.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_183.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_184.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_185.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_186.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_187.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_188.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_189.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_19.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_19.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_190.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_191.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_192.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_193.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_194.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_195.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_196.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_197.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_198.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_199.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_2.ab.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_2.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_2.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_20.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_20.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_200.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_201.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_202.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_203.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_204.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_205.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_206.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_207.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_208.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_209.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_21.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_21.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_210.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_211.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_212.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_213.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_214.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_215.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_216.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_217.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_218.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_219.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_22.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_22.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_220.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_221.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_222.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_223.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_224.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_225.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_226.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_227.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_228.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_229.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_23.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_23.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_230.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_231.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_232.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_233.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_234.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_235.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_236.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_237.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_238.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_239.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_24.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_24.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_240.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_241.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_242.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_243.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_244.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_245.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_246.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_247.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_248.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_249.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_25.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_25.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_250.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_251.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_252.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_253.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_254.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_255.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_256.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_257.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_258.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_259.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_26.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_26.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_260.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_261.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_262.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_263.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_264.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_265.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_266.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_267.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_268.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_269.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_27.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_27.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_270.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_271.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_272.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_273.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_274.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_275.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_276.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_277.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_278.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_279.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_28.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_28.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_280.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_281.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_282.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_283.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_284.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_285.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_286.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_287.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_288.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_289.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_29.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_29.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_290.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_291.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_292.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_293.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_294.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_295.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_296.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_297.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_298.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_299.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_3.ab.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_3.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_3.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_30.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_30.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_300.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_301.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_302.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_303.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_304.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_305.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_306.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_307.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_308.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_309.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_31.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_310.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_311.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_312.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_313.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_314.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_315.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_316.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_317.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_318.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_319.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_32.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_320.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_321.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_322.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_323.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_324.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_325.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_326.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_327.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_328.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_329.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_33.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_330.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_331.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_332.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_333.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_334.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_335.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_336.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_337.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_338.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_339.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_34.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_340.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_341.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_342.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_343.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_344.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_345.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_346.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_347.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_348.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_349.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_35.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_350.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_351.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_352.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_353.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_354.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_355.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_356.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_36.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_37.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_38.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_39.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_4.ab.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_4.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_4.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_40.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_41.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_42.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_43.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_44.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_45.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_46.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_47.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_48.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_49.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_5.ab.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_5.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_5.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_50.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_51.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_52.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_53.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_54.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_55.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_56.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_57.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_58.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_59.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_6.ab.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_6.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_6.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_60.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_61.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_62.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_63.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_64.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_65.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_66.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_67.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_68.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_69.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_7.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_7.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_70.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_71.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_72.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_73.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_74.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_75.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_76.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_77.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_78.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_79.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_8.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_8.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_80.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_81.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_82.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_83.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_84.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_85.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_86.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_87.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_88.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_89.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_9.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_9.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_90.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_91.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_92.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_93.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_94.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_95.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_96.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_97.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_98.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/8_99.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_1.ab.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_1.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_1.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_10.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_10.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_100.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_101.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_102.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_103.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_104.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_105.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_106.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_107.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_108.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_109.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_11.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_11.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_110.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_111.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_112.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_113.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_114.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_115.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_116.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_117.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_118.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_119.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_12.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_12.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_120.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_121.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_122.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_123.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_124.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_125.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_126.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_127.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_128.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_129.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_13.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_13.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_130.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_131.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_132.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_133.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_134.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_135.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_136.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_137.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_138.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_139.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_14.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_14.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_140.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_141.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_142.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_143.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_144.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_145.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_146.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_147.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_148.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_149.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_15.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_15.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_150.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_151.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_152.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_153.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_154.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_155.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_156.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_157.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_158.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_159.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_16.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_16.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_160.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_161.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_162.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_163.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_164.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_165.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_166.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_167.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_168.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_169.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_17.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_17.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_170.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_171.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_172.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_173.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_174.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_175.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_176.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_177.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_178.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_179.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_18.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_18.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_180.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_181.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_182.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_183.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_184.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_185.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_186.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_187.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_188.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_189.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_19.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_19.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_190.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_191.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_192.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_193.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_194.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_195.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_196.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_197.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_198.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_199.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_2.ab.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_2.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_2.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_20.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_20.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_200.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_201.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_202.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_203.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_204.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_205.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_206.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_207.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_208.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_209.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_21.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_21.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_210.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_211.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_212.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_213.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_214.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_215.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_216.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_217.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_218.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_219.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_22.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_22.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_220.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_221.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_222.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_223.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_224.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_225.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_226.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_227.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_228.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_229.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_23.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_230.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_231.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_232.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_233.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_234.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_235.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_236.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_237.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_238.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_239.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_24.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_240.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_241.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_242.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_243.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_244.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_245.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_246.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_247.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_248.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_249.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_25.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_250.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_251.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_252.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_253.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_254.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_255.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_256.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_257.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_258.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_259.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_26.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_260.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_261.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_262.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_263.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_264.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_265.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_266.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_267.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_268.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_269.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_27.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_270.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_271.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_272.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_28.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_29.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_3.ab.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_3.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_3.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_30.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_31.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_32.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_33.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_34.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_35.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_36.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_37.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_38.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_39.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_4.ab.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_4.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_4.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_40.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_41.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_42.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_43.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_44.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_45.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_46.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_47.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_48.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_49.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_5.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_5.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_50.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_51.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_52.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_53.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_54.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_55.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_56.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_57.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_58.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_59.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_6.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_6.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_60.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_61.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_62.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_63.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_64.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_65.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_66.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_67.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_68.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_69.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_7.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_7.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_70.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_71.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_72.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_73.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_74.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_75.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_76.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_77.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_78.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_79.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_8.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_8.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_80.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_81.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_82.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_83.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_84.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_85.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_86.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_87.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_88.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_89.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_9.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_9.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_90.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_91.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_92.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_93.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_94.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_95.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_96.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_97.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_98.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/eng_Latn/9_99.cc.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/eng_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/eng_Latn.md5"}
{"files": 6, "bytes": 2389417494, "documents": 805976, "segments": 25197270, "tokens": 1349847907, "characters": 4036578463, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["md", "openlid_v2", "ts", "f", "seg_langs_openlid_v3", "id", "rs", "pii", "openlid_v3", "s", "glotlid_v3", "filter", "cluster_size", "xml", "crawl_id", "web-register", "de", "doc_scores", "u", "o", "c", "text"], "optional": ["htmllang", "allowed", "metalang"]}, "name": "epo_Latn", "t/d": 1674.7991342173962, "c/t": 2.9903950230742553, "samples": "https://data.hplt-project.org/three/sorted/epo_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/epo_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/epo_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/epo_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/epo_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/epo_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/epo_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/epo_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/epo_Latn.md5"}
{"files": 6, "bytes": 6563551676, "documents": 3556988, "segments": 60033566, "tokens": 3479016117, "characters": 10409016833, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["text", "xml", "md", "f", "o", "s", "rs", "u", "c", "ts", "de", "id", "openlid_v2", "glotlid_v3", "openlid_v3", "seg_langs_openlid_v3", "cluster_size", "web-register", "crawl_id", "filter", "pii", "doc_scores", "bsc-edu", "finepdfs-edu", "jql", "propella-4b"], "optional": ["htmllang", "metalang", "allowed"]}, "name": "eus_Latn", "t/d": 978.0792392327442, "c/t": 2.991942688088444, "samples": "https://data.hplt-project.org/three/sorted/eus_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/eus_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/eus_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/eus_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/eus_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/eus_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/eus_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/eus_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/eus_Latn.md5"}
{"files": 6, "bytes": 23626904, "documents": 7431, "segments": 220467, "tokens": 19017627, "characters": 40764612, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["rs", "s", "openlid_v2", "seg_langs_openlid_v3", "c", "doc_scores", "de", "md", "openlid_v3", "cluster_size", "u", "filter", "pii", "id", "xml", "o", "crawl_id", "ts", "f", "glotlid_v3", "text"], "optional": ["htmllang", "allowed", "metalang"]}, "name": "ewe_Latn", "t/d": 2559.228502220428, "c/t": 2.143517274789331, "samples": "https://data.hplt-project.org/three/sorted/ewe_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/ewe_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ewe_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ewe_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ewe_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ewe_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ewe_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/ewe_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/ewe_Latn.md5"}
{"files": 6, "bytes": 435009270, "documents": 349930, "segments": 5668804, "tokens": 286611046, "characters": 741762055, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["u", "md", "text", "glotlid_v3", "rs", "f", "filter", "seg_langs_openlid_v3", "openlid_v2", "xml", "c", "ts", "pii", "de", "cluster_size", "o", "crawl_id", "s", "openlid_v3", "id", "doc_scores"], "optional": ["metalang", "allowed", "htmllang"]}, "name": "fao_Latn", "t/d": 819.0525133598148, "c/t": 2.58804419910599, "samples": "https://data.hplt-project.org/three/sorted/fao_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/fao_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/fao_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/fao_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/fao_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/fao_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/fao_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/fao_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/fao_Latn.md5"}
{"files": 33, "bytes": 368942045325, "documents": 134811762, "segments": 3918657155, "tokens": 168822721491, "characters": 514525328958, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["xml", "rs", "s", "md", "de", "ts", "crawl_id", "id", "f", "seg_langs_openlid_v3", "openlid_v3", "filter", "cluster_size", "openlid_v2", "o", "c", "pii", "doc_scores", "web-register", "u", "text", "glotlid_v3"], "optional": ["htmllang", "metalang", "allowed"]}, "name": "fas_Arab", "t/d": 1252.28480799027, "c/t": 3.0477255929405778, "samples": "https://data.hplt-project.org/three/sorted/fas_Arab.json", "urls": ["https://data.hplt-project.org/three/sorted/fas_Arab/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/fas_Arab/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/fas_Arab/5_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/fas_Arab/5_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/fas_Arab/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/fas_Arab/6_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/fas_Arab/6_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/fas_Arab/6_4.jsonl.zst", "https://data.hplt-project.org/three/sorted/fas_Arab/6_5.jsonl.zst", "https://data.hplt-project.org/three/sorted/fas_Arab/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/fas_Arab/7_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/fas_Arab/7_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/fas_Arab/7_4.jsonl.zst", "https://data.hplt-project.org/three/sorted/fas_Arab/7_5.jsonl.zst", "https://data.hplt-project.org/three/sorted/fas_Arab/7_6.jsonl.zst", "https://data.hplt-project.org/three/sorted/fas_Arab/7_7.jsonl.zst", "https://data.hplt-project.org/three/sorted/fas_Arab/7_8.jsonl.zst", "https://data.hplt-project.org/three/sorted/fas_Arab/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/fas_Arab/8_10.jsonl.zst", "https://data.hplt-project.org/three/sorted/fas_Arab/8_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/fas_Arab/8_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/fas_Arab/8_4.jsonl.zst", "https://data.hplt-project.org/three/sorted/fas_Arab/8_5.jsonl.zst", "https://data.hplt-project.org/three/sorted/fas_Arab/8_6.jsonl.zst", "https://data.hplt-project.org/three/sorted/fas_Arab/8_7.jsonl.zst", "https://data.hplt-project.org/three/sorted/fas_Arab/8_8.jsonl.zst", "https://data.hplt-project.org/three/sorted/fas_Arab/8_9.jsonl.zst", "https://data.hplt-project.org/three/sorted/fas_Arab/9_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/fas_Arab/9_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/fas_Arab/9_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/fas_Arab/9_4.jsonl.zst", "https://data.hplt-project.org/three/sorted/fas_Arab/9_5.jsonl.zst", "https://data.hplt-project.org/three/sorted/fas_Arab/9_6.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/fas_Arab.map", "md5": "https://data.hplt-project.org/three/sorted/fas_Arab.md5"}
{"files": 6, "bytes": 25130152, "documents": 12399, "segments": 281234, "tokens": 19314068, "characters": 54478254, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["de", "id", "openlid_v2", "cluster_size", "md", "doc_scores", "c", "seg_langs_openlid_v3", "o", "filter", "u", "text", "s", "openlid_v3", "rs", "pii", "xml", "glotlid_v3", "f", "ts", "crawl_id"], "optional": ["allowed", "metalang", "htmllang"]}, "name": "fij_Latn", "t/d": 1557.711750947657, "c/t": 2.8206514546805987, "samples": "https://data.hplt-project.org/three/sorted/fij_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/fij_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/fij_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/fij_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/fij_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/fij_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/fij_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/fij_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/fij_Latn.md5"}
{"files": 6, "bytes": 8051793751, "documents": 3762020, "segments": 88794475, "tokens": 4427745736, "characters": 15257036412, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["ts", "openlid_v3", "text", "pii", "f", "xml", "filter", "rs", "cluster_size", "o", "openlid_v2", "crawl_id", "id", "glotlid_v3", "md", "web-register", "c", "doc_scores", "u", "seg_langs_openlid_v3", "de", "s"], "optional": ["htmllang", "metalang", "allowed"]}, "name": "fil_Latn", "t/d": 1176.9596482740656, "c/t": 3.4457797086115245, "samples": "https://data.hplt-project.org/three/sorted/fil_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/fil_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/fil_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/fil_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/fil_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/fil_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/fil_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/fil_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/fil_Latn.md5"}
{"files": 10, "bytes": 120112180074, "documents": 51435563, "segments": 1233318040, "tokens": 67056983952, "characters": 198204262032, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["text", "xml", "md", "f", "o", "s", "rs", "u", "c", "ts", "de", "id", "openlid_v2", "glotlid_v3", "openlid_v3", "seg_langs_openlid_v3", "cluster_size", "web-register", "crawl_id", "filter", "pii", "doc_scores", "bsc-edu", "finepdfs-edu", "jql", "propella-4b"], "optional": ["htmllang", "metalang", "allowed"]}, "name": "fin_Latn", "t/d": 1303.7085635088704, "c/t": 2.9557586749484055, "samples": "https://data.hplt-project.org/three/sorted/fin_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/fin_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/fin_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/fin_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/fin_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/fin_Latn/7_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/fin_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/fin_Latn/8_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/fin_Latn/8_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/fin_Latn/9_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/fin_Latn/9_2.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/fin_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/fin_Latn.md5"}
{"files": 5, "bytes": 2989685, "documents": 1524, "segments": 22722, "tokens": 2961536, "characters": 5478541, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["filter", "id", "rs", "doc_scores", "de", "ts", "seg_langs_openlid_v3", "openlid_v2", "glotlid_v3", "cluster_size", "c", "openlid_v3", "text", "md", "f", "crawl_id", "s", "xml", "o", "pii", "u"], "optional": ["metalang", "htmllang", "allowed"]}, "name": "fon_Latn", "t/d": 1943.265091863517, "c/t": 1.849898498616934, "samples": "https://data.hplt-project.org/three/sorted/fon_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/fon_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/fon_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/fon_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/fon_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/fon_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/fon_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/fon_Latn.md5"}
{"files": 139, "bytes": 2214442567015, "documents": 914929291, "segments": 24126954703, "tokens": 908371974194, "characters": 3551270478477, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["text", "xml", "md", "f", "o", "s", "rs", "u", "c", "ts", "de", "id", "openlid_v2", "glotlid_v3", "openlid_v3", "seg_langs_openlid_v3", "cluster_size", "web-register", "crawl_id", "filter", "pii", "doc_scores", "bsc-edu", "finepdfs-edu", "fineweb2-hq", "jql", "propella-4b"], "optional": ["metalang", "htmllang", "allowed"]}, "name": "fra_Latn", "t/d": 992.8329796952582, "c/t": 3.9094892614097305, "samples": "https://data.hplt-project.org/three/sorted/fra_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/fra_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/10_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/5_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/5_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/5_4.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/5_5.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/5_6.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/5_7.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/5_8.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/5_9.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/6_10.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/6_11.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/6_12.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/6_13.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/6_14.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/6_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/6_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/6_4.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/6_5.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/6_6.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/6_7.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/6_8.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/6_9.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/7_10.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/7_11.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/7_12.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/7_13.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/7_14.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/7_15.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/7_16.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/7_17.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/7_18.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/7_19.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/7_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/7_20.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/7_21.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/7_22.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/7_23.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/7_24.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/7_25.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/7_26.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/7_27.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/7_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/7_4.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/7_5.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/7_6.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/7_7.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/7_8.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/7_9.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/8_10.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/8_11.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/8_12.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/8_13.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/8_14.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/8_15.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/8_16.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/8_17.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/8_18.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/8_19.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/8_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/8_20.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/8_21.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/8_22.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/8_23.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/8_24.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/8_25.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/8_26.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/8_27.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/8_28.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/8_29.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/8_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/8_30.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/8_31.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/8_32.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/8_33.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/8_34.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/8_35.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/8_36.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/8_37.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/8_38.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/8_39.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/8_4.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/8_40.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/8_41.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/8_5.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/8_6.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/8_7.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/8_8.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/8_9.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/9_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/9_10.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/9_11.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/9_12.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/9_13.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/9_14.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/9_15.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/9_16.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/9_17.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/9_18.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/9_19.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/9_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/9_20.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/9_21.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/9_22.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/9_23.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/9_24.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/9_25.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/9_26.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/9_27.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/9_28.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/9_29.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/9_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/9_30.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/9_31.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/9_32.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/9_33.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/9_34.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/9_35.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/9_36.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/9_37.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/9_38.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/9_39.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/9_4.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/9_40.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/9_41.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/9_42.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/9_43.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/9_44.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/9_45.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/9_46.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/9_5.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/9_6.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/9_7.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/9_8.jsonl.zst", "https://data.hplt-project.org/three/sorted/fra_Latn/9_9.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/fra_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/fra_Latn.md5"}
{"files": 5, "bytes": 7147259, "documents": 5475, "segments": 105134, "tokens": 4405657, "characters": 11676017, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["md", "s", "de", "openlid_v3", "ts", "id", "doc_scores", "text", "openlid_v2", "u", "xml", "o", "c", "rs", "f", "filter", "cluster_size", "glotlid_v3", "seg_langs_openlid_v3", "pii", "crawl_id"], "optional": ["metalang", "htmllang", "allowed"]}, "name": "frp_Latn", "t/d": 804.686210045662, "c/t": 2.650232871056462, "samples": "https://data.hplt-project.org/three/sorted/frp_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/frp_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/frp_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/frp_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/frp_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/frp_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/frp_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/frp_Latn.md5"}
{"files": 6, "bytes": 104352697, "documents": 64874, "segments": 1268106, "tokens": 83397770, "characters": 250919800, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["glotlid_v3", "ts", "c", "rs", "crawl_id", "pii", "o", "s", "openlid_v3", "md", "openlid_v2", "f", "cluster_size", "u", "doc_scores", "de", "id", "filter", "xml", "text", "seg_langs_openlid_v3"], "optional": ["htmllang", "metalang", "allowed"]}, "name": "fur_Latn", "t/d": 1285.5345747140611, "c/t": 3.0087111441948626, "samples": "https://data.hplt-project.org/three/sorted/fur_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/fur_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/fur_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/fur_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/fur_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/fur_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/fur_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/fur_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/fur_Latn.md5"}
{"files": 6, "bytes": 9199069, "documents": 6755, "segments": 82020, "tokens": 7017141, "characters": 16955273, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["rs", "filter", "openlid_v3", "seg_langs_openlid_v3", "cluster_size", "o", "ts", "text", "openlid_v2", "glotlid_v3", "xml", "pii", "f", "doc_scores", "crawl_id", "id", "c", "md", "de", "u", "s"], "optional": ["allowed", "htmllang", "metalang"]}, "name": "fuv_Latn", "t/d": 1038.8069578090303, "c/t": 2.4162651142395455, "samples": "https://data.hplt-project.org/three/sorted/fuv_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/fuv_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/fuv_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/fuv_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/fuv_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/fuv_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/fuv_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/fuv_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/fuv_Latn.md5"}
{"files": 6, "bytes": 112667878, "documents": 60293, "segments": 995967, "tokens": 80269627, "characters": 219622097, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["u", "seg_langs_openlid_v3", "de", "md", "f", "openlid_v3", "glotlid_v3", "id", "ts", "doc_scores", "s", "text", "xml", "rs", "openlid_v2", "crawl_id", "pii", "o", "filter", "c", "cluster_size", "web-register"], "optional": ["metalang", "allowed", "htmllang"]}, "name": "gaz_Latn", "t/d": 1331.325808966215, "c/t": 2.7360547844578873, "samples": "https://data.hplt-project.org/three/sorted/gaz_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/gaz_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/gaz_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/gaz_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/gaz_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/gaz_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/gaz_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/gaz_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/gaz_Latn.md5"}
{"files": 6, "bytes": 845321341, "documents": 323596, "segments": 8837350, "tokens": 619732529, "characters": 1727400353, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["f", "openlid_v3", "ts", "openlid_v2", "u", "web-register", "s", "c", "text", "glotlid_v3", "md", "o", "doc_scores", "id", "de", "rs", "seg_langs_openlid_v3", "crawl_id", "pii", "filter", "xml", "cluster_size"], "optional": ["htmllang", "metalang", "allowed"]}, "name": "gla_Latn", "t/d": 1915.1427366222078, "c/t": 2.787332070155059, "samples": "https://data.hplt-project.org/three/sorted/gla_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/gla_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/gla_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/gla_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/gla_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/gla_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/gla_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/gla_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/gla_Latn.md5"}
{"files": 6, "bytes": 2065988792, "documents": 885117, "segments": 21528930, "tokens": 1318799316, "characters": 3586535105, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["text", "xml", "md", "f", "o", "s", "rs", "u", "c", "ts", "de", "id", "openlid_v2", "glotlid_v3", "openlid_v3", "seg_langs_openlid_v3", "cluster_size", "web-register", "crawl_id", "filter", "pii", "doc_scores", "bsc-edu", "jql", "propella-4b"], "optional": ["htmllang", "metalang", "allowed"]}, "name": "gle_Latn", "t/d": 1489.9717393293768, "c/t": 2.719545772800507, "samples": "https://data.hplt-project.org/three/sorted/gle_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/gle_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/gle_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/gle_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/gle_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/gle_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/gle_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/gle_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/gle_Latn.md5"}
{"files": 6, "bytes": 7623377837, "documents": 4102297, "segments": 66541997, "tokens": 3134786698, "characters": 11772802410, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["text", "xml", "md", "f", "o", "s", "rs", "u", "c", "ts", "de", "id", "openlid_v2", "glotlid_v3", "openlid_v3", "seg_langs_openlid_v3", "cluster_size", "web-register", "crawl_id", "filter", "pii", "doc_scores", "bsc-edu", "finepdfs-edu", "jql", "propella-4b"], "optional": ["allowed", "metalang", "htmllang"]}, "name": "glg_Latn", "t/d": 764.154008839438, "c/t": 3.7555353981535875, "samples": "https://data.hplt-project.org/three/sorted/glg_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/glg_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/glg_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/glg_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/glg_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/glg_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/glg_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/glg_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/glg_Latn.md5"}
{"files": 6, "bytes": 19634599, "documents": 10809, "segments": 253825, "tokens": 14056577, "characters": 35710423, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["id", "cluster_size", "o", "text", "u", "crawl_id", "rs", "filter", "openlid_v2", "xml", "openlid_v3", "s", "seg_langs_openlid_v3", "f", "doc_scores", "de", "glotlid_v3", "c", "pii", "md", "ts"], "optional": ["metalang", "allowed", "htmllang"]}, "name": "gug_Latn", "t/d": 1300.4511980756777, "c/t": 2.5404778844806954, "samples": "https://data.hplt-project.org/three/sorted/gug_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/gug_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/gug_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/gug_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/gug_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/gug_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/gug_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/gug_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/gug_Latn.md5"}
{"files": 6, "bytes": 8216633375, "documents": 3771646, "segments": 51526768, "tokens": 3683481414, "characters": 9231902134, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["pii", "openlid_v3", "cluster_size", "xml", "rs", "ts", "md", "web-register", "id", "s", "doc_scores", "de", "f", "seg_langs_openlid_v3", "crawl_id", "openlid_v2", "filter", "o", "c", "u", "glotlid_v3", "text"], "optional": ["allowed", "metalang", "htmllang"]}, "name": "guj_Gujr", "t/d": 976.6243740796459, "c/t": 2.5062980089737463, "samples": "https://data.hplt-project.org/three/sorted/guj_Gujr.json", "urls": ["https://data.hplt-project.org/three/sorted/guj_Gujr/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/guj_Gujr/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/guj_Gujr/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/guj_Gujr/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/guj_Gujr/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/guj_Gujr/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/guj_Gujr.map", "md5": "https://data.hplt-project.org/three/sorted/guj_Gujr.md5"}
{"files": 6, "bytes": 669333341, "documents": 402357, "segments": 8215465, "tokens": 411085434, "characters": 1198200020, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["ts", "openlid_v3", "filter", "de", "pii", "crawl_id", "id", "glotlid_v3", "seg_langs_openlid_v3", "md", "f", "xml", "u", "text", "s", "doc_scores", "c", "o", "openlid_v2", "cluster_size", "rs"], "optional": ["allowed", "htmllang", "metalang"]}, "name": "hat_Latn", "t/d": 1021.6932574802973, "c/t": 2.9147226364629595, "samples": "https://data.hplt-project.org/three/sorted/hat_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/hat_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/hat_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/hat_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/hat_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/hat_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/hat_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/hat_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/hat_Latn.md5"}
{"files": 6, "bytes": 1459478587, "documents": 786114, "segments": 16634199, "tokens": 883053859, "characters": 2598337509, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["openlid_v3", "f", "xml", "u", "filter", "seg_langs_openlid_v3", "text", "glotlid_v3", "web-register", "s", "de", "rs", "pii", "ts", "doc_scores", "o", "openlid_v2", "md", "id", "crawl_id", "c", "cluster_size"], "optional": ["metalang", "allowed", "htmllang"]}, "name": "hau_Latn", "t/d": 1123.315268523395, "c/t": 2.9424451096815827, "samples": "https://data.hplt-project.org/three/sorted/hau_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/hau_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/hau_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/hau_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/hau_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/hau_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/hau_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/hau_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/hau_Latn.md5"}
{"files": 8, "bytes": 69895943895, "documents": 28615796, "segments": 699895831, "tokens": 40739532137, "characters": 87164740075, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["f", "pii", "c", "cluster_size", "web-register", "u", "glotlid_v3", "s", "seg_langs_openlid_v3", "de", "xml", "ts", "o", "crawl_id", "filter", "id", "md", "rs", "text", "openlid_v3", "openlid_v2", "doc_scores"], "optional": ["htmllang", "allowed", "metalang"]}, "name": "heb_Hebr", "t/d": 1423.672860157376, "c/t": 2.1395616371312283, "samples": "https://data.hplt-project.org/three/sorted/heb_Hebr.json", "urls": ["https://data.hplt-project.org/three/sorted/heb_Hebr/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/heb_Hebr/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/heb_Hebr/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/heb_Hebr/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/heb_Hebr/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/heb_Hebr/8_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/heb_Hebr/9_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/heb_Hebr/9_2.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/heb_Hebr.map", "md5": "https://data.hplt-project.org/three/sorted/heb_Hebr.md5"}
{"files": 11, "bytes": 90370975905, "documents": 41084098, "segments": 556735444, "tokens": 27713802829, "characters": 103823846087, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["de", "u", "openlid_v2", "text", "crawl_id", "doc_scores", "f", "seg_langs_openlid_v3", "rs", "md", "openlid_v3", "cluster_size", "ts", "xml", "pii", "web-register", "id", "c", "o", "glotlid_v3", "s", "filter"], "optional": ["htmllang", "metalang", "allowed"]}, "name": "hin_Deva", "t/d": 674.5627670589239, "c/t": 3.746286524718928, "samples": "https://data.hplt-project.org/three/sorted/hin_Deva.json", "urls": ["https://data.hplt-project.org/three/sorted/hin_Deva/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/hin_Deva/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/hin_Deva/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/hin_Deva/6_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/hin_Deva/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/hin_Deva/7_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/hin_Deva/7_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/hin_Deva/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/hin_Deva/8_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/hin_Deva/8_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/hin_Deva/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/hin_Deva.map", "md5": "https://data.hplt-project.org/three/sorted/hin_Deva.md5"}
{"files": 5, "bytes": 16891024, "documents": 7721, "segments": 94488, "tokens": 7567713, "characters": 21146828, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["rs", "seg_langs_openlid_v3", "o", "filter", "xml", "doc_scores", "crawl_id", "glotlid_v3", "pii", "s", "de", "ts", "f", "text", "c", "openlid_v3", "u", "cluster_size", "md", "openlid_v2", "id"], "optional": ["htmllang", "allowed", "metalang"]}, "name": "hne_Deva", "t/d": 980.1467426499158, "c/t": 2.794348570036945, "samples": "https://data.hplt-project.org/three/sorted/hne_Deva.json", "urls": ["https://data.hplt-project.org/three/sorted/hne_Deva/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/hne_Deva/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/hne_Deva/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/hne_Deva/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/hne_Deva/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/hne_Deva.map", "md5": "https://data.hplt-project.org/three/sorted/hne_Deva.md5"}
{"files": 6, "bytes": 43289181500, "documents": 17447507, "segments": 426011635, "tokens": 22810473857, "characters": 71924795281, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["text", "xml", "md", "f", "o", "s", "rs", "u", "c", "ts", "de", "id", "openlid_v2", "glotlid_v3", "openlid_v3", "seg_langs_openlid_v3", "cluster_size", "crawl_id", "filter", "pii", "doc_scores", "bsc-edu", "finepdfs-edu", "jql", "propella-4b"], "optional": ["allowed", "htmllang", "metalang"]}, "name": "hrv_Latn", "t/d": 1307.3772577939071, "c/t": 3.1531477921896816, "samples": "https://data.hplt-project.org/three/sorted/hrv_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/hrv_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/hrv_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/hrv_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/hrv_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/hrv_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/hrv_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/hrv_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/hrv_Latn.md5"}
{"files": 15, "bytes": 194476056176, "documents": 80685036, "segments": 1758387035, "tokens": 104633671452, "characters": 303539062802, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["text", "xml", "md", "f", "o", "s", "rs", "u", "c", "ts", "de", "id", "openlid_v2", "glotlid_v3", "openlid_v3", "seg_langs_openlid_v3", "cluster_size", "web-register", "crawl_id", "filter", "pii", "doc_scores", "bsc-edu", "finepdfs-edu", "fineweb2-hq", "jql", "propella-4b"], "optional": ["htmllang", "metalang", "allowed"]}, "name": "hun_Latn", "t/d": 1296.8163198439918, "c/t": 2.900969244314881, "samples": "https://data.hplt-project.org/three/sorted/hun_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/hun_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/hun_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/hun_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/hun_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/hun_Latn/7_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/hun_Latn/7_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/hun_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/hun_Latn/8_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/hun_Latn/8_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/hun_Latn/8_4.jsonl.zst", "https://data.hplt-project.org/three/sorted/hun_Latn/9_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/hun_Latn/9_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/hun_Latn/9_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/hun_Latn/9_4.jsonl.zst", "https://data.hplt-project.org/three/sorted/hun_Latn/9_5.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/hun_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/hun_Latn.md5"}
{"files": 6, "bytes": 12053208156, "documents": 6373266, "segments": 104379028, "tokens": 9194338655, "characters": 17016598834, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["crawl_id", "glotlid_v3", "doc_scores", "id", "de", "pii", "o", "cluster_size", "ts", "seg_langs_openlid_v3", "rs", "openlid_v3", "text", "u", "web-register", "s", "openlid_v2", "xml", "f", "filter", "c", "md"], "optional": ["allowed", "htmllang", "metalang"]}, "name": "hye_Armn", "t/d": 1442.6415992993232, "c/t": 1.8507692040194923, "samples": "https://data.hplt-project.org/three/sorted/hye_Armn.json", "urls": ["https://data.hplt-project.org/three/sorted/hye_Armn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/hye_Armn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/hye_Armn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/hye_Armn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/hye_Armn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/hye_Armn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/hye_Armn.map", "md5": "https://data.hplt-project.org/three/sorted/hye_Armn.md5"}
{"files": 6, "bytes": 510930018, "documents": 217355, "segments": 5723774, "tokens": 406612823, "characters": 953422861, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["md", "crawl_id", "text", "de", "u", "openlid_v2", "seg_langs_openlid_v3", "openlid_v3", "ts", "cluster_size", "pii", "c", "rs", "glotlid_v3", "filter", "o", "id", "xml", "f", "doc_scores", "s"], "optional": ["metalang", "allowed", "htmllang"]}, "name": "ibo_Latn", "t/d": 1870.7313979434566, "c/t": 2.3447929014280007, "samples": "https://data.hplt-project.org/three/sorted/ibo_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/ibo_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ibo_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ibo_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ibo_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ibo_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ibo_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/ibo_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/ibo_Latn.md5"}
{"files": 6, "bytes": 68989163, "documents": 45272, "segments": 831541, "tokens": 43636219, "characters": 132651483, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["cluster_size", "de", "rs", "id", "filter", "crawl_id", "openlid_v2", "o", "f", "c", "doc_scores", "u", "xml", "pii", "md", "openlid_v3", "text", "ts", "seg_langs_openlid_v3", "s", "glotlid_v3"], "optional": ["metalang", "allowed", "htmllang"]}, "name": "ilo_Latn", "t/d": 963.8677107262768, "c/t": 3.03993989488411, "samples": "https://data.hplt-project.org/three/sorted/ilo_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/ilo_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ilo_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ilo_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ilo_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ilo_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ilo_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/ilo_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/ilo_Latn.md5"}
{"files": 26, "bytes": 359264903742, "documents": 192999338, "segments": 3747165857, "tokens": 150998974541, "characters": 654945192233, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["c", "md", "openlid_v2", "openlid_v3", "o", "s", "seg_langs_openlid_v3", "f", "rs", "cluster_size", "glotlid_v3", "web-register", "pii", "doc_scores", "id", "ts", "xml", "filter", "de", "text", "u", "crawl_id"], "optional": ["allowed", "metalang", "htmllang"]}, "name": "ind_Latn", "t/d": 782.3807900366995, "c/t": 4.337414834927677, "samples": "https://data.hplt-project.org/three/sorted/ind_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/ind_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ind_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ind_Latn/5_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/ind_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ind_Latn/6_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/ind_Latn/6_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/ind_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ind_Latn/7_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/ind_Latn/7_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/ind_Latn/7_4.jsonl.zst", "https://data.hplt-project.org/three/sorted/ind_Latn/7_5.jsonl.zst", "https://data.hplt-project.org/three/sorted/ind_Latn/7_6.jsonl.zst", "https://data.hplt-project.org/three/sorted/ind_Latn/7_7.jsonl.zst", "https://data.hplt-project.org/three/sorted/ind_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ind_Latn/8_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/ind_Latn/8_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/ind_Latn/8_4.jsonl.zst", "https://data.hplt-project.org/three/sorted/ind_Latn/8_5.jsonl.zst", "https://data.hplt-project.org/three/sorted/ind_Latn/8_6.jsonl.zst", "https://data.hplt-project.org/three/sorted/ind_Latn/8_7.jsonl.zst", "https://data.hplt-project.org/three/sorted/ind_Latn/8_8.jsonl.zst", "https://data.hplt-project.org/three/sorted/ind_Latn/8_9.jsonl.zst", "https://data.hplt-project.org/three/sorted/ind_Latn/9_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ind_Latn/9_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/ind_Latn/9_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/ind_Latn/9_4.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/ind_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/ind_Latn.md5"}
{"files": 6, "bytes": 9058747555, "documents": 4546258, "segments": 91365464, "tokens": 5574680800, "characters": 13996485560, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["text", "xml", "md", "f", "o", "s", "rs", "u", "c", "ts", "de", "id", "openlid_v2", "glotlid_v3", "openlid_v3", "seg_langs_openlid_v3", "cluster_size", "web-register", "crawl_id", "filter", "pii", "doc_scores", "bsc-edu", "finepdfs-edu", "jql", "propella-4b"], "optional": ["htmllang", "metalang", "allowed"]}, "name": "isl_Latn", "t/d": 1226.2130305847138, "c/t": 2.5107241225363075, "samples": "https://data.hplt-project.org/three/sorted/isl_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/isl_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/isl_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/isl_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/isl_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/isl_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/isl_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/isl_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/isl_Latn.md5"}
{"files": 52, "bytes": 816916475140, "documents": 384192861, "segments": 7214118463, "tokens": 326720810648, "characters": 1274032624073, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["text", "xml", "md", "f", "o", "s", "rs", "u", "c", "ts", "de", "id", "openlid_v2", "glotlid_v3", "openlid_v3", "seg_langs_openlid_v3", "cluster_size", "web-register", "crawl_id", "filter", "pii", "doc_scores", "bsc-edu", "finepdfs-edu", "fineweb2-hq", "jql", "propella-4b"], "optional": ["htmllang", "metalang", "allowed"]}, "name": "ita_Latn", "t/d": 850.408333454171, "c/t": 3.8994535473456806, "samples": "https://data.hplt-project.org/three/sorted/ita_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/ita_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ita_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ita_Latn/5_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/ita_Latn/5_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/ita_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ita_Latn/6_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/ita_Latn/6_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/ita_Latn/6_4.jsonl.zst", "https://data.hplt-project.org/three/sorted/ita_Latn/6_5.jsonl.zst", "https://data.hplt-project.org/three/sorted/ita_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ita_Latn/7_10.jsonl.zst", "https://data.hplt-project.org/three/sorted/ita_Latn/7_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/ita_Latn/7_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/ita_Latn/7_4.jsonl.zst", "https://data.hplt-project.org/three/sorted/ita_Latn/7_5.jsonl.zst", "https://data.hplt-project.org/three/sorted/ita_Latn/7_6.jsonl.zst", "https://data.hplt-project.org/three/sorted/ita_Latn/7_7.jsonl.zst", "https://data.hplt-project.org/three/sorted/ita_Latn/7_8.jsonl.zst", "https://data.hplt-project.org/three/sorted/ita_Latn/7_9.jsonl.zst", "https://data.hplt-project.org/three/sorted/ita_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ita_Latn/8_10.jsonl.zst", "https://data.hplt-project.org/three/sorted/ita_Latn/8_11.jsonl.zst", "https://data.hplt-project.org/three/sorted/ita_Latn/8_12.jsonl.zst", "https://data.hplt-project.org/three/sorted/ita_Latn/8_13.jsonl.zst", "https://data.hplt-project.org/three/sorted/ita_Latn/8_14.jsonl.zst", "https://data.hplt-project.org/three/sorted/ita_Latn/8_15.jsonl.zst", "https://data.hplt-project.org/three/sorted/ita_Latn/8_16.jsonl.zst", "https://data.hplt-project.org/three/sorted/ita_Latn/8_17.jsonl.zst", "https://data.hplt-project.org/three/sorted/ita_Latn/8_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/ita_Latn/8_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/ita_Latn/8_4.jsonl.zst", "https://data.hplt-project.org/three/sorted/ita_Latn/8_5.jsonl.zst", "https://data.hplt-project.org/three/sorted/ita_Latn/8_6.jsonl.zst", "https://data.hplt-project.org/three/sorted/ita_Latn/8_7.jsonl.zst", "https://data.hplt-project.org/three/sorted/ita_Latn/8_8.jsonl.zst", "https://data.hplt-project.org/three/sorted/ita_Latn/8_9.jsonl.zst", "https://data.hplt-project.org/three/sorted/ita_Latn/9_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ita_Latn/9_10.jsonl.zst", "https://data.hplt-project.org/three/sorted/ita_Latn/9_11.jsonl.zst", "https://data.hplt-project.org/three/sorted/ita_Latn/9_12.jsonl.zst", "https://data.hplt-project.org/three/sorted/ita_Latn/9_13.jsonl.zst", "https://data.hplt-project.org/three/sorted/ita_Latn/9_14.jsonl.zst", "https://data.hplt-project.org/three/sorted/ita_Latn/9_15.jsonl.zst", "https://data.hplt-project.org/three/sorted/ita_Latn/9_16.jsonl.zst", "https://data.hplt-project.org/three/sorted/ita_Latn/9_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/ita_Latn/9_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/ita_Latn/9_4.jsonl.zst", "https://data.hplt-project.org/three/sorted/ita_Latn/9_5.jsonl.zst", "https://data.hplt-project.org/three/sorted/ita_Latn/9_6.jsonl.zst", "https://data.hplt-project.org/three/sorted/ita_Latn/9_7.jsonl.zst", "https://data.hplt-project.org/three/sorted/ita_Latn/9_8.jsonl.zst", "https://data.hplt-project.org/three/sorted/ita_Latn/9_9.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/ita_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/ita_Latn.md5"}
{"files": 6, "bytes": 740831434, "documents": 347413, "segments": 9113026, "tokens": 417613645, "characters": 1401957628, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["u", "f", "text", "o", "c", "xml", "openlid_v3", "md", "pii", "filter", "openlid_v2", "id", "web-register", "doc_scores", "rs", "crawl_id", "ts", "s", "de", "glotlid_v3", "cluster_size", "seg_langs_openlid_v3"], "optional": ["htmllang", "metalang", "allowed"]}, "name": "jav_Latn", "t/d": 1202.0668339987278, "c/t": 3.3570685363980384, "samples": "https://data.hplt-project.org/three/sorted/jav_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/jav_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/jav_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/jav_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/jav_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/jav_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/jav_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/jav_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/jav_Latn.md5"}
{"files": 176, "bytes": 2803665399628, "documents": 841786271, "segments": 47455959150, "tokens": 1127508139993, "characters": 1920861280556, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["u", "id", "f", "ts", "cluster_size", "de", "text", "o", "s", "pii", "rs", "doc_scores", "md", "openlid_v2", "xml", "filter", "web-register", "crawl_id", "c", "seg_langs_openlid_v3", "glotlid_v3", "openlid_v3"], "optional": ["allowed", "htmllang", "metalang"]}, "name": "jpn_Jpan", "t/d": 1339.4232940548873, "c/t": 1.7036340691677183, "samples": "https://data.hplt-project.org/three/sorted/jpn_Jpan.json", "urls": ["https://data.hplt-project.org/three/sorted/jpn_Jpan/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/5_10.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/5_11.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/5_12.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/5_13.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/5_14.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/5_15.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/5_16.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/5_17.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/5_18.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/5_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/5_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/5_4.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/5_5.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/5_6.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/5_7.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/5_8.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/5_9.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/6_10.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/6_11.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/6_12.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/6_13.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/6_14.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/6_15.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/6_16.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/6_17.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/6_18.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/6_19.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/6_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/6_20.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/6_21.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/6_22.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/6_23.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/6_24.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/6_25.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/6_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/6_4.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/6_5.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/6_6.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/6_7.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/6_8.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/6_9.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/7_10.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/7_11.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/7_12.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/7_13.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/7_14.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/7_15.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/7_16.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/7_17.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/7_18.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/7_19.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/7_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/7_20.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/7_21.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/7_22.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/7_23.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/7_24.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/7_25.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/7_26.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/7_27.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/7_28.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/7_29.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/7_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/7_30.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/7_31.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/7_32.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/7_33.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/7_34.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/7_35.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/7_36.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/7_37.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/7_38.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/7_39.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/7_4.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/7_40.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/7_41.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/7_42.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/7_43.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/7_44.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/7_45.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/7_46.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/7_47.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/7_48.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/7_49.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/7_5.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/7_50.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/7_51.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/7_52.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/7_53.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/7_54.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/7_55.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/7_6.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/7_7.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/7_8.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/7_9.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/8_10.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/8_11.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/8_12.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/8_13.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/8_14.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/8_15.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/8_16.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/8_17.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/8_18.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/8_19.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/8_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/8_20.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/8_21.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/8_22.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/8_23.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/8_24.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/8_25.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/8_26.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/8_27.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/8_28.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/8_29.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/8_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/8_30.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/8_31.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/8_32.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/8_33.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/8_34.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/8_35.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/8_36.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/8_37.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/8_38.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/8_39.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/8_4.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/8_40.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/8_41.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/8_42.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/8_43.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/8_44.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/8_45.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/8_46.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/8_47.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/8_48.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/8_49.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/8_5.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/8_50.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/8_6.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/8_7.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/8_8.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/8_9.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/9_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/9_10.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/9_11.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/9_12.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/9_13.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/9_14.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/9_15.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/9_16.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/9_17.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/9_18.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/9_19.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/9_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/9_20.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/9_21.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/9_22.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/9_23.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/9_24.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/9_25.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/9_26.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/9_27.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/9_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/9_4.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/9_5.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/9_6.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/9_7.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/9_8.jsonl.zst", "https://data.hplt-project.org/three/sorted/jpn_Jpan/9_9.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/jpn_Jpan.map", "md5": "https://data.hplt-project.org/three/sorted/jpn_Jpan.md5"}
{"files": 6, "bytes": 29689989, "documents": 14579, "segments": 377131, "tokens": 21335214, "characters": 48952658, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["text", "seg_langs_openlid_v3", "rs", "o", "id", "pii", "ts", "openlid_v3", "xml", "glotlid_v3", "doc_scores", "f", "openlid_v2", "de", "md", "crawl_id", "filter", "s", "c", "cluster_size", "u"], "optional": ["allowed", "metalang", "htmllang"]}, "name": "kab_Latn", "t/d": 1463.4209479388162, "c/t": 2.2944535733271763, "samples": "https://data.hplt-project.org/three/sorted/kab_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/kab_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/kab_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/kab_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/kab_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/kab_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/kab_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/kab_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/kab_Latn.md5"}
{"files": 6, "bytes": 12315551, "documents": 8749, "segments": 141022, "tokens": 9738380, "characters": 27063610, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["de", "md", "seg_langs_openlid_v3", "c", "text", "cluster_size", "ts", "openlid_v2", "filter", "f", "id", "crawl_id", "glotlid_v3", "openlid_v3", "s", "xml", "pii", "rs", "doc_scores", "u", "o"], "optional": ["allowed", "metalang", "htmllang"]}, "name": "kac_Latn", "t/d": 1113.0849239913132, "c/t": 2.779066949533701, "samples": "https://data.hplt-project.org/three/sorted/kac_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/kac_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/kac_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/kac_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/kac_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/kac_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/kac_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/kac_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/kac_Latn.md5"}
{"files": 5, "bytes": 1661162, "documents": 812, "segments": 15494, "tokens": 1566130, "characters": 3144616, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["s", "ts", "filter", "doc_scores", "f", "seg_langs_openlid_v3", "cluster_size", "id", "rs", "xml", "openlid_v2", "c", "openlid_v3", "glotlid_v3", "u", "crawl_id", "pii", "md", "o", "text", "de"], "optional": ["allowed", "htmllang", "metalang"]}, "name": "kam_Latn", "t/d": 1928.731527093596, "c/t": 2.0078895110878405, "samples": "https://data.hplt-project.org/three/sorted/kam_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/kam_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/kam_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/kam_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/kam_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/kam_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/kam_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/kam_Latn.md5"}
{"files": 6, "bytes": 9406160636, "documents": 4763526, "segments": 60833658, "tokens": 4195963824, "characters": 10739931015, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["rs", "id", "openlid_v2", "seg_langs_openlid_v3", "xml", "text", "pii", "crawl_id", "f", "openlid_v3", "glotlid_v3", "c", "u", "doc_scores", "md", "de", "cluster_size", "s", "web-register", "ts", "o", "filter"], "optional": ["htmllang", "allowed", "metalang"]}, "name": "kan_Knda", "t/d": 880.852507995128, "c/t": 2.559586179835472, "samples": "https://data.hplt-project.org/three/sorted/kan_Knda.json", "urls": ["https://data.hplt-project.org/three/sorted/kan_Knda/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/kan_Knda/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/kan_Knda/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/kan_Knda/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/kan_Knda/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/kan_Knda/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/kan_Knda.map", "md5": "https://data.hplt-project.org/three/sorted/kan_Knda.md5"}
{"files": 5, "bytes": 3177859, "documents": 1612, "segments": 30448, "tokens": 2123006, "characters": 4247042, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["text", "crawl_id", "u", "de", "xml", "f", "rs", "openlid_v3", "glotlid_v3", "pii", "cluster_size", "openlid_v2", "id", "ts", "filter", "doc_scores", "s", "seg_langs_openlid_v3", "c", "o", "md"], "optional": ["metalang", "htmllang", "allowed"]}, "name": "kas_Arab", "t/d": 1317.001240694789, "c/t": 2.0004851611347307, "samples": "https://data.hplt-project.org/three/sorted/kas_Arab.json", "urls": ["https://data.hplt-project.org/three/sorted/kas_Arab/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/kas_Arab/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/kas_Arab/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/kas_Arab/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/kas_Arab/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/kas_Arab.map", "md5": "https://data.hplt-project.org/three/sorted/kas_Arab.md5"}
{"files": 5, "bytes": 60559, "documents": 29, "segments": 272, "tokens": 32563, "characters": 63047, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["o", "f", "id", "cluster_size", "pii", "crawl_id", "md", "filter", "doc_scores", "u", "rs", "s", "de", "text", "glotlid_v3", "c", "openlid_v2", "xml", "openlid_v3", "ts", "seg_langs_openlid_v3"], "optional": ["metalang", "htmllang", "allowed"]}, "name": "kas_Deva", "t/d": 1122.8620689655172, "c/t": 1.9361545312164112, "samples": "https://data.hplt-project.org/three/sorted/kas_Deva.json", "urls": ["https://data.hplt-project.org/three/sorted/kas_Deva/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/kas_Deva/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/kas_Deva/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/kas_Deva/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/kas_Deva/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/kas_Deva.map", "md5": "https://data.hplt-project.org/three/sorted/kas_Deva.md5"}
{"files": 6, "bytes": 17640376495, "documents": 6809381, "segments": 113166793, "tokens": 8279088334, "characters": 18682810419, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["text", "xml", "md", "f", "o", "s", "rs", "u", "c", "ts", "de", "id", "openlid_v2", "glotlid_v3", "openlid_v3", "seg_langs_openlid_v3", "cluster_size", "web-register", "crawl_id", "filter", "pii", "doc_scores", "bsc-edu", "finepdfs-edu", "jql", "propella-4b"], "optional": ["htmllang", "metalang", "allowed"]}, "name": "kat_Geor", "t/d": 1215.83567346283, "c/t": 2.2566265348655237, "samples": "https://data.hplt-project.org/three/sorted/kat_Geor.json", "urls": ["https://data.hplt-project.org/three/sorted/kat_Geor/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/kat_Geor/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/kat_Geor/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/kat_Geor/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/kat_Geor/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/kat_Geor/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/kat_Geor.map", "md5": "https://data.hplt-project.org/three/sorted/kat_Geor.md5"}
{"files": 6, "bytes": 13303516984, "documents": 5847858, "segments": 110292087, "tokens": 8117246447, "characters": 19052923542, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["openlid_v3", "filter", "id", "openlid_v2", "doc_scores", "md", "u", "s", "pii", "xml", "crawl_id", "text", "f", "web-register", "c", "rs", "o", "ts", "de", "cluster_size", "glotlid_v3", "seg_langs_openlid_v3"], "optional": ["htmllang", "allowed", "metalang"]}, "name": "kaz_Cyrl", "t/d": 1388.0717430211198, "c/t": 2.3472151136967936, "samples": "https://data.hplt-project.org/three/sorted/kaz_Cyrl.json", "urls": ["https://data.hplt-project.org/three/sorted/kaz_Cyrl/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/kaz_Cyrl/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/kaz_Cyrl/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/kaz_Cyrl/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/kaz_Cyrl/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/kaz_Cyrl/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/kaz_Cyrl.map", "md5": "https://data.hplt-project.org/three/sorted/kaz_Cyrl.md5"}
{"files": 5, "bytes": 5380895, "documents": 3369, "segments": 41072, "tokens": 6168746, "characters": 9913314, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["s", "cluster_size", "u", "xml", "md", "f", "text", "doc_scores", "openlid_v2", "rs", "id", "glotlid_v3", "ts", "openlid_v3", "pii", "filter", "o", "c", "de", "crawl_id", "seg_langs_openlid_v3"], "optional": ["htmllang", "allowed", "metalang"]}, "name": "kbp_Latn", "t/d": 1831.0317601662214, "c/t": 1.607022561797811, "samples": "https://data.hplt-project.org/three/sorted/kbp_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/kbp_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/kbp_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/kbp_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/kbp_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/kbp_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/kbp_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/kbp_Latn.md5"}
{"files": 6, "bytes": 4202887, "documents": 3020, "segments": 51023, "tokens": 2334671, "characters": 6978878, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["text", "openlid_v2", "f", "de", "pii", "md", "o", "glotlid_v3", "cluster_size", "filter", "rs", "id", "s", "ts", "crawl_id", "openlid_v3", "xml", "seg_langs_openlid_v3", "c", "doc_scores", "u"], "optional": ["metalang", "htmllang", "allowed"]}, "name": "kea_Latn", "t/d": 773.0698675496689, "c/t": 2.989234029120163, "samples": "https://data.hplt-project.org/three/sorted/kea_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/kea_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/kea_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/kea_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/kea_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/kea_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/kea_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/kea_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/kea_Latn.md5"}
{"files": 6, "bytes": 8551489085, "documents": 3535011, "segments": 76168103, "tokens": 6225840067, "characters": 13283412497, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["openlid_v3", "id", "crawl_id", "f", "u", "filter", "md", "c", "doc_scores", "cluster_size", "o", "web-register", "rs", "seg_langs_openlid_v3", "s", "xml", "text", "glotlid_v3", "de", "pii", "openlid_v2", "ts"], "optional": ["allowed", "htmllang", "metalang"]}, "name": "khk_Cyrl", "t/d": 1761.1939728051766, "c/t": 2.133593596052778, "samples": "https://data.hplt-project.org/three/sorted/khk_Cyrl.json", "urls": ["https://data.hplt-project.org/three/sorted/khk_Cyrl/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/khk_Cyrl/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/khk_Cyrl/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/khk_Cyrl/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/khk_Cyrl/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/khk_Cyrl/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/khk_Cyrl.map", "md5": "https://data.hplt-project.org/three/sorted/khk_Cyrl.md5"}
{"files": 6, "bytes": 605749152, "documents": 229636, "segments": 3395819, "tokens": 345587181, "characters": 671736744, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["id", "md", "o", "cluster_size", "filter", "seg_langs_openlid_v3", "rs", "web-register", "crawl_id", "u", "f", "openlid_v2", "xml", "glotlid_v3", "text", "pii", "ts", "doc_scores", "openlid_v3", "de", "c", "s"], "optional": ["htmllang", "allowed", "metalang"]}, "name": "khm_Khmr", "t/d": 1504.9346835861973, "c/t": 1.9437548061135983, "samples": "https://data.hplt-project.org/three/sorted/khm_Khmr.json", "urls": ["https://data.hplt-project.org/three/sorted/khm_Khmr/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/khm_Khmr/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/khm_Khmr/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/khm_Khmr/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/khm_Khmr/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/khm_Khmr/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/khm_Khmr.map", "md5": "https://data.hplt-project.org/three/sorted/khm_Khmr.md5"}
{"files": 5, "bytes": 11299715, "documents": 8018, "segments": 123692, "tokens": 8867740, "characters": 20057108, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["doc_scores", "cluster_size", "u", "xml", "rs", "openlid_v3", "id", "glotlid_v3", "o", "seg_langs_openlid_v3", "c", "s", "f", "pii", "openlid_v2", "filter", "md", "crawl_id", "ts", "de", "text"], "optional": ["htmllang", "allowed", "metalang"]}, "name": "kik_Latn", "t/d": 1105.9790471439262, "c/t": 2.2618060520493386, "samples": "https://data.hplt-project.org/three/sorted/kik_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/kik_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/kik_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/kik_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/kik_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/kik_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/kik_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/kik_Latn.md5"}
{"files": 6, "bytes": 863516763, "documents": 581364, "segments": 9134349, "tokens": 592141609, "characters": 1621198821, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["crawl_id", "rs", "c", "text", "s", "pii", "ts", "glotlid_v3", "md", "filter", "o", "u", "cluster_size", "id", "openlid_v3", "seg_langs_openlid_v3", "doc_scores", "de", "f", "openlid_v2", "xml"], "optional": ["metalang", "allowed", "htmllang"]}, "name": "kin_Latn", "t/d": 1018.5384870752231, "c/t": 2.737856614632869, "samples": "https://data.hplt-project.org/three/sorted/kin_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/kin_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/kin_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/kin_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/kin_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/kin_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/kin_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/kin_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/kin_Latn.md5"}
{"files": 6, "bytes": 2709325464, "documents": 1543966, "segments": 20495766, "tokens": 1477878718, "characters": 3745965620, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["crawl_id", "web-register", "openlid_v3", "glotlid_v3", "text", "de", "c", "xml", "md", "doc_scores", "filter", "f", "rs", "s", "id", "pii", "openlid_v2", "cluster_size", "seg_langs_openlid_v3", "o", "ts", "u"], "optional": ["htmllang", "metalang", "allowed"]}, "name": "kir_Cyrl", "t/d": 957.196413651596, "c/t": 2.534690820278799, "samples": "https://data.hplt-project.org/three/sorted/kir_Cyrl.json", "urls": ["https://data.hplt-project.org/three/sorted/kir_Cyrl/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/kir_Cyrl/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/kir_Cyrl/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/kir_Cyrl/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/kir_Cyrl/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/kir_Cyrl/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/kir_Cyrl.map", "md5": "https://data.hplt-project.org/three/sorted/kir_Cyrl.md5"}
{"files": 6, "bytes": 2283031, "documents": 1165, "segments": 20717, "tokens": 1892936, "characters": 4863228, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["rs", "u", "openlid_v2", "pii", "xml", "ts", "c", "cluster_size", "text", "seg_langs_openlid_v3", "id", "crawl_id", "openlid_v3", "glotlid_v3", "filter", "f", "de", "s", "md", "doc_scores", "o"], "optional": ["metalang", "allowed", "htmllang"]}, "name": "kmb_Latn", "t/d": 1624.8377682403434, "c/t": 2.569145496731004, "samples": "https://data.hplt-project.org/three/sorted/kmb_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/kmb_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/kmb_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/kmb_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/kmb_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/kmb_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/kmb_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/kmb_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/kmb_Latn.md5"}
{"files": 6, "bytes": 1297966187, "documents": 743276, "segments": 12839120, "tokens": 851576167, "characters": 2116985598, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["c", "o", "openlid_v3", "filter", "rs", "ts", "openlid_v2", "cluster_size", "glotlid_v3", "id", "f", "de", "seg_langs_openlid_v3", "pii", "xml", "s", "web-register", "doc_scores", "md", "u", "crawl_id", "text"], "optional": ["metalang", "allowed", "htmllang"]}, "name": "kmr_Latn", "t/d": 1145.7065302794656, "c/t": 2.485961538188515, "samples": "https://data.hplt-project.org/three/sorted/kmr_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/kmr_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/kmr_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/kmr_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/kmr_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/kmr_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/kmr_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/kmr_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/kmr_Latn.md5"}
{"files": 5, "bytes": 726338, "documents": 110, "segments": 24909, "tokens": 763720, "characters": 1102734, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["rs", "ts", "text", "f", "o", "doc_scores", "seg_langs_openlid_v3", "c", "u", "crawl_id", "filter", "openlid_v2", "id", "md", "de", "xml", "glotlid_v3", "openlid_v3", "pii", "s", "cluster_size"], "optional": ["htmllang", "metalang", "allowed"]}, "name": "knc_Arab", "t/d": 6942.909090909091, "c/t": 1.4438982873304353, "samples": "https://data.hplt-project.org/three/sorted/knc_Arab.json", "urls": ["https://data.hplt-project.org/three/sorted/knc_Arab/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/knc_Arab/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/knc_Arab/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/knc_Arab/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/knc_Arab/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/knc_Arab.map", "md5": "https://data.hplt-project.org/three/sorted/knc_Arab.md5"}
{"files": 5, "bytes": 1723563, "documents": 1078, "segments": 14741, "tokens": 1316881, "characters": 3309309, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["c", "de", "glotlid_v3", "pii", "o", "md", "seg_langs_openlid_v3", "s", "id", "filter", "openlid_v3", "rs", "f", "u", "doc_scores", "text", "crawl_id", "xml", "ts", "openlid_v2", "cluster_size"], "optional": ["metalang", "htmllang"]}, "name": "knc_Latn", "t/d": 1221.5964749536179, "c/t": 2.512990163879652, "samples": "https://data.hplt-project.org/three/sorted/knc_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/knc_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/knc_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/knc_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/knc_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/knc_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/knc_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/knc_Latn.md5"}
{"files": 16, "bytes": 215340359103, "documents": 81544455, "segments": 2481785522, "tokens": 104401546417, "characters": 176927370431, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["de", "text", "pii", "s", "openlid_v3", "cluster_size", "id", "ts", "c", "u", "glotlid_v3", "web-register", "crawl_id", "rs", "doc_scores", "seg_langs_openlid_v3", "o", "md", "f", "openlid_v2", "xml", "filter"], "optional": ["allowed", "metalang", "htmllang"]}, "name": "kor_Hang", "t/d": 1280.302215730058, "c/t": 1.694681510983734, "samples": "https://data.hplt-project.org/three/sorted/kor_Hang.json", "urls": ["https://data.hplt-project.org/three/sorted/kor_Hang/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/kor_Hang/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/kor_Hang/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/kor_Hang/6_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/kor_Hang/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/kor_Hang/7_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/kor_Hang/7_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/kor_Hang/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/kor_Hang/8_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/kor_Hang/8_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/kor_Hang/8_4.jsonl.zst", "https://data.hplt-project.org/three/sorted/kor_Hang/8_5.jsonl.zst", "https://data.hplt-project.org/three/sorted/kor_Hang/9_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/kor_Hang/9_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/kor_Hang/9_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/kor_Hang/9_4.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/kor_Hang.map", "md5": "https://data.hplt-project.org/three/sorted/kor_Hang.md5"}
{"files": 5, "bytes": 307588, "documents": 411, "segments": 3739, "tokens": 184174, "characters": 513398, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["filter", "s", "text", "openlid_v2", "md", "cluster_size", "xml", "ts", "seg_langs_openlid_v3", "f", "crawl_id", "c", "openlid_v3", "id", "rs", "doc_scores", "de", "pii", "u", "glotlid_v3", "o"], "optional": ["htmllang", "metalang", "allowed"]}, "name": "ktu_Latn", "t/d": 448.11192214111924, "c/t": 2.7875704496834515, "samples": "https://data.hplt-project.org/three/sorted/ktu_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/ktu_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ktu_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ktu_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ktu_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ktu_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/ktu_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/ktu_Latn.md5"}
{"files": 4, "bytes": 165013574, "documents": 38362, "segments": 634245, "tokens": 135253350, "characters": 166681244, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["c", "o", "seg_langs_openlid_v3", "glotlid_v3", "filter", "openlid_v3", "openlid_v2", "xml", "crawl_id", "id", "de", "text", "f", "s", "web-register", "doc_scores", "md", "rs", "pii", "cluster_size", "ts", "u"], "optional": ["allowed", "htmllang", "metalang"]}, "name": "lao_Laoo", "t/d": 3525.711641728794, "c/t": 1.2323631466429483, "samples": "https://data.hplt-project.org/three/sorted/lao_Laoo.json", "urls": ["https://data.hplt-project.org/three/sorted/lao_Laoo/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/lao_Laoo/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/lao_Laoo/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/lao_Laoo/8_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/lao_Laoo.map", "md5": "https://data.hplt-project.org/three/sorted/lao_Laoo.md5"}
{"files": 6, "bytes": 1444900372, "documents": 1234302, "segments": 15978205, "tokens": 643815408, "characters": 2340351530, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["f", "xml", "md", "pii", "text", "cluster_size", "openlid_v2", "c", "u", "glotlid_v3", "doc_scores", "o", "ts", "openlid_v3", "rs", "s", "de", "seg_langs_openlid_v3", "id", "crawl_id", "filter"], "optional": ["allowed", "metalang", "htmllang"]}, "name": "lat_Latn", "t/d": 521.6028232960815, "c/t": 3.635128176366975, "samples": "https://data.hplt-project.org/three/sorted/lat_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/lat_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/lat_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/lat_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/lat_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/lat_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/lat_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/lat_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/lat_Latn.md5"}
{"files": 5, "bytes": 15930456, "documents": 6746, "segments": 252134, "tokens": 9734672, "characters": 24863628, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["de", "text", "id", "cluster_size", "glotlid_v3", "openlid_v3", "o", "filter", "rs", "xml", "crawl_id", "openlid_v2", "md", "pii", "doc_scores", "ts", "c", "f", "seg_langs_openlid_v3", "u", "s"], "optional": ["htmllang", "allowed", "metalang"]}, "name": "lij_Latn", "t/d": 1443.0287577823897, "c/t": 2.5541310482777435, "samples": "https://data.hplt-project.org/three/sorted/lij_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/lij_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/lij_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/lij_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/lij_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/lij_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/lij_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/lij_Latn.md5"}
{"files": 6, "bytes": 93293720, "documents": 55038, "segments": 1455552, "tokens": 53079607, "characters": 146948168, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["glotlid_v3", "o", "seg_langs_openlid_v3", "md", "f", "openlid_v2", "u", "doc_scores", "xml", "rs", "id", "c", "crawl_id", "s", "de", "ts", "cluster_size", "text", "filter", "pii", "openlid_v3"], "optional": ["metalang", "htmllang", "allowed"]}, "name": "lim_Latn", "t/d": 964.4174388604237, "c/t": 2.76844868124212, "samples": "https://data.hplt-project.org/three/sorted/lim_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/lim_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/lim_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/lim_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/lim_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/lim_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/lim_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/lim_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/lim_Latn.md5"}
{"files": 6, "bytes": 34691109, "documents": 13566, "segments": 394728, "tokens": 24054515, "characters": 72047265, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["f", "xml", "doc_scores", "rs", "openlid_v3", "pii", "id", "c", "de", "seg_langs_openlid_v3", "u", "s", "text", "openlid_v2", "md", "o", "cluster_size", "ts", "glotlid_v3", "filter", "crawl_id"], "optional": ["allowed", "metalang", "htmllang"]}, "name": "lin_Latn", "t/d": 1773.1472062509215, "c/t": 2.9951659802743893, "samples": "https://data.hplt-project.org/three/sorted/lin_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/lin_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/lin_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/lin_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/lin_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/lin_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/lin_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/lin_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/lin_Latn.md5"}
{"files": 8, "bytes": 53710805262, "documents": 21946435, "segments": 535817028, "tokens": 30315427110, "characters": 85075229311, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["text", "xml", "md", "f", "o", "s", "rs", "u", "c", "ts", "de", "id", "openlid_v2", "glotlid_v3", "openlid_v3", "seg_langs_openlid_v3", "cluster_size", "web-register", "crawl_id", "filter", "pii", "doc_scores", "bsc-edu", "finepdfs-edu", "jql", "propella-4b"], "optional": ["htmllang", "metalang", "allowed"]}, "name": "lit_Latn", "t/d": 1381.3372016912997, "c/t": 2.8063345108846134, "samples": "https://data.hplt-project.org/three/sorted/lit_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/lit_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/lit_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/lit_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/lit_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/lit_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/lit_Latn/8_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/lit_Latn/9_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/lit_Latn/9_2.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/lit_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/lit_Latn.md5"}
{"files": 6, "bytes": 45728127, "documents": 26096, "segments": 540757, "tokens": 25642587, "characters": 71819825, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["rs", "text", "id", "doc_scores", "de", "o", "pii", "cluster_size", "filter", "seg_langs_openlid_v3", "openlid_v3", "f", "xml", "ts", "crawl_id", "s", "openlid_v2", "md", "u", "glotlid_v3", "c"], "optional": ["htmllang", "allowed", "metalang"]}, "name": "lmo_Latn", "t/d": 982.6251916002452, "c/t": 2.8008026257257117, "samples": "https://data.hplt-project.org/three/sorted/lmo_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/lmo_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/lmo_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/lmo_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/lmo_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/lmo_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/lmo_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/lmo_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/lmo_Latn.md5"}
{"files": 6, "bytes": 26933353, "documents": 14111, "segments": 214650, "tokens": 18821142, "characters": 46363764, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["text", "xml", "md", "f", "o", "s", "rs", "u", "c", "ts", "de", "id", "openlid_v2", "glotlid_v3", "openlid_v3", "seg_langs_openlid_v3", "cluster_size", "crawl_id", "filter", "pii", "doc_scores", "bsc-edu", "finepdfs-edu", "jql", "propella-4b"], "optional": ["htmllang", "metalang", "allowed"]}, "name": "ltg_Latn", "t/d": 1333.7922188363689, "c/t": 2.463387397002796, "samples": "https://data.hplt-project.org/three/sorted/ltg_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/ltg_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ltg_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ltg_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ltg_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ltg_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ltg_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/ltg_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/ltg_Latn.md5"}
{"files": 6, "bytes": 850173870, "documents": 467065, "segments": 9418945, "tokens": 501297831, "characters": 1552140437, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["id", "openlid_v3", "c", "cluster_size", "s", "pii", "rs", "f", "seg_langs_openlid_v3", "o", "u", "doc_scores", "de", "filter", "glotlid_v3", "crawl_id", "xml", "ts", "openlid_v2", "text", "md"], "optional": ["allowed", "metalang", "htmllang"]}, "name": "ltz_Latn", "t/d": 1073.2935051866443, "c/t": 3.0962440709223813, "samples": "https://data.hplt-project.org/three/sorted/ltz_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/ltz_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ltz_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ltz_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ltz_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ltz_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ltz_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/ltz_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/ltz_Latn.md5"}
{"files": 6, "bytes": 5353656, "documents": 1770, "segments": 51181, "tokens": 4405257, "characters": 12090858, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["filter", "o", "u", "f", "openlid_v3", "doc_scores", "pii", "s", "glotlid_v3", "crawl_id", "c", "text", "xml", "ts", "de", "openlid_v2", "rs", "cluster_size", "id", "seg_langs_openlid_v3", "md"], "optional": ["allowed", "htmllang", "metalang"]}, "name": "lua_Latn", "t/d": 2488.8457627118646, "c/t": 2.7446430480673434, "samples": "https://data.hplt-project.org/three/sorted/lua_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/lua_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/lua_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/lua_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/lua_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/lua_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/lua_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/lua_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/lua_Latn.md5"}
{"files": 6, "bytes": 65500382, "documents": 49637, "segments": 753756, "tokens": 45575916, "characters": 120277069, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["doc_scores", "seg_langs_openlid_v3", "cluster_size", "c", "de", "text", "s", "openlid_v3", "u", "o", "pii", "filter", "f", "openlid_v2", "id", "rs", "glotlid_v3", "crawl_id", "ts", "xml", "md"], "optional": ["metalang", "allowed", "htmllang"]}, "name": "lug_Latn", "t/d": 918.1843382960292, "c/t": 2.6390488564179377, "samples": "https://data.hplt-project.org/three/sorted/lug_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/lug_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/lug_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/lug_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/lug_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/lug_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/lug_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/lug_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/lug_Latn.md5"}
{"files": 5, "bytes": 5955310, "documents": 2541, "segments": 68264, "tokens": 4451731, "characters": 12102330, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["c", "f", "de", "filter", "doc_scores", "ts", "crawl_id", "rs", "cluster_size", "text", "seg_langs_openlid_v3", "xml", "s", "openlid_v2", "id", "pii", "md", "u", "o", "glotlid_v3", "openlid_v3"], "optional": ["metalang", "allowed", "htmllang"]}, "name": "luo_Latn", "t/d": 1751.9602518693428, "c/t": 2.7185672269955217, "samples": "https://data.hplt-project.org/three/sorted/luo_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/luo_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/luo_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/luo_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/luo_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/luo_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/luo_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/luo_Latn.md5"}
{"files": 6, "bytes": 267132455, "documents": 173032, "segments": 3094655, "tokens": 185556135, "characters": 535634205, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["filter", "rs", "doc_scores", "openlid_v3", "pii", "c", "f", "openlid_v2", "md", "seg_langs_openlid_v3", "s", "de", "glotlid_v3", "xml", "u", "o", "text", "ts", "crawl_id", "cluster_size", "id"], "optional": ["allowed", "metalang", "htmllang"]}, "name": "lus_Latn", "t/d": 1072.3804556382634, "c/t": 2.8866423899161298, "samples": "https://data.hplt-project.org/three/sorted/lus_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/lus_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/lus_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/lus_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/lus_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/lus_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/lus_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/lus_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/lus_Latn.md5"}
{"files": 6, "bytes": 29484594464, "documents": 12157413, "segments": 305950529, "tokens": 17929744121, "characters": 46383921056, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["text", "xml", "md", "f", "o", "s", "rs", "u", "c", "ts", "de", "id", "openlid_v2", "glotlid_v3", "openlid_v3", "seg_langs_openlid_v3", "cluster_size", "web-register", "crawl_id", "filter", "pii", "doc_scores", "bsc-edu", "finepdfs-edu", "jql", "propella-4b"], "optional": ["allowed", "htmllang", "metalang"]}, "name": "lvs_Latn", "t/d": 1474.7992949651377, "c/t": 2.5869817629841902, "samples": "https://data.hplt-project.org/three/sorted/lvs_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/lvs_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/lvs_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/lvs_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/lvs_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/lvs_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/lvs_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/lvs_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/lvs_Latn.md5"}
{"files": 4, "bytes": 2902283, "documents": 524, "segments": 62964, "tokens": 1581945, "characters": 4102943, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["openlid_v3", "crawl_id", "xml", "seg_langs_openlid_v3", "u", "f", "de", "openlid_v2", "id", "doc_scores", "text", "filter", "ts", "glotlid_v3", "pii", "o", "rs", "cluster_size", "c", "s", "md"], "optional": ["allowed", "htmllang", "metalang"]}, "name": "mag_Deva", "t/d": 3018.9790076335876, "c/t": 2.59360660452797, "samples": "https://data.hplt-project.org/three/sorted/mag_Deva.json", "urls": ["https://data.hplt-project.org/three/sorted/mag_Deva/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/mag_Deva/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/mag_Deva/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/mag_Deva/8_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/mag_Deva.map", "md5": "https://data.hplt-project.org/three/sorted/mag_Deva.md5"}
{"files": 6, "bytes": 95248659, "documents": 30264, "segments": 830213, "tokens": 42348733, "characters": 112913513, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["id", "text", "f", "crawl_id", "seg_langs_openlid_v3", "o", "c", "doc_scores", "de", "pii", "md", "u", "glotlid_v3", "s", "xml", "cluster_size", "filter", "openlid_v2", "rs", "ts", "openlid_v3"], "optional": ["metalang", "htmllang", "allowed"]}, "name": "mai_Deva", "t/d": 1399.3105009251917, "c/t": 2.6662784220722733, "samples": "https://data.hplt-project.org/three/sorted/mai_Deva.json", "urls": ["https://data.hplt-project.org/three/sorted/mai_Deva/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/mai_Deva/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/mai_Deva/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/mai_Deva/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/mai_Deva/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/mai_Deva/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/mai_Deva.map", "md5": "https://data.hplt-project.org/three/sorted/mai_Deva.md5"}
{"files": 6, "bytes": 16605732470, "documents": 8536691, "segments": 90647992, "tokens": 6783985174, "characters": 19456919299, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["u", "web-register", "rs", "ts", "s", "xml", "cluster_size", "glotlid_v3", "doc_scores", "text", "f", "de", "seg_langs_openlid_v3", "c", "pii", "o", "crawl_id", "openlid_v3", "md", "filter", "id", "openlid_v2"], "optional": ["allowed", "htmllang", "metalang"]}, "name": "mal_Mlym", "t/d": 794.6855724308166, "c/t": 2.8680663061543417, "samples": "https://data.hplt-project.org/three/sorted/mal_Mlym.json", "urls": ["https://data.hplt-project.org/three/sorted/mal_Mlym/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/mal_Mlym/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/mal_Mlym/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/mal_Mlym/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/mal_Mlym/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/mal_Mlym/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/mal_Mlym.map", "md5": "https://data.hplt-project.org/three/sorted/mal_Mlym.md5"}
{"files": 6, "bytes": 15596443607, "documents": 7273220, "segments": 90974450, "tokens": 5168550447, "characters": 17948728882, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["cluster_size", "ts", "c", "filter", "openlid_v2", "xml", "text", "seg_langs_openlid_v3", "f", "md", "crawl_id", "rs", "pii", "de", "openlid_v3", "web-register", "id", "glotlid_v3", "u", "s", "o", "doc_scores"], "optional": ["htmllang", "metalang", "allowed"]}, "name": "mar_Deva", "t/d": 710.627541446567, "c/t": 3.472681376732628, "samples": "https://data.hplt-project.org/three/sorted/mar_Deva.json", "urls": ["https://data.hplt-project.org/three/sorted/mar_Deva/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/mar_Deva/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/mar_Deva/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/mar_Deva/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/mar_Deva/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/mar_Deva/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/mar_Deva.map", "md5": "https://data.hplt-project.org/three/sorted/mar_Deva.md5"}
{"files": 6, "bytes": 25695405, "documents": 13019, "segments": 322899, "tokens": 15159783, "characters": 47448483, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["cluster_size", "xml", "crawl_id", "o", "id", "md", "s", "seg_langs_openlid_v3", "de", "c", "u", "text", "pii", "doc_scores", "f", "glotlid_v3", "ts", "openlid_v2", "rs", "openlid_v3", "filter"], "optional": ["metalang", "htmllang", "allowed"]}, "name": "min_Latn", "t/d": 1164.4352868883939, "c/t": 3.1298919648124253, "samples": "https://data.hplt-project.org/three/sorted/min_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/min_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/min_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/min_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/min_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/min_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/min_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/min_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/min_Latn.md5"}
{"files": 6, "bytes": 14093534982, "documents": 7424445, "segments": 105191100, "tokens": 6410787882, "characters": 17779175414, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["text", "xml", "md", "f", "o", "s", "rs", "u", "c", "ts", "de", "id", "openlid_v2", "glotlid_v3", "openlid_v3", "seg_langs_openlid_v3", "cluster_size", "web-register", "crawl_id", "filter", "pii", "doc_scores", "bsc-edu", "finepdfs-edu", "jql", "propella-4b"], "optional": ["htmllang", "metalang", "allowed"]}, "name": "mkd_Cyrl", "t/d": 863.4703175792938, "c/t": 2.773321429635784, "samples": "https://data.hplt-project.org/three/sorted/mkd_Cyrl.json", "urls": ["https://data.hplt-project.org/three/sorted/mkd_Cyrl/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/mkd_Cyrl/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/mkd_Cyrl/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/mkd_Cyrl/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/mkd_Cyrl/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/mkd_Cyrl/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/mkd_Cyrl.map", "md5": "https://data.hplt-project.org/three/sorted/mkd_Cyrl.md5"}
{"files": 6, "bytes": 1813384188, "documents": 854014, "segments": 21523245, "tokens": 1275724798, "characters": 3198455520, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["text", "xml", "md", "f", "o", "s", "rs", "u", "c", "ts", "de", "id", "openlid_v2", "glotlid_v3", "openlid_v3", "seg_langs_openlid_v3", "cluster_size", "crawl_id", "filter", "pii", "doc_scores", "bsc-edu", "finepdfs-edu", "jql", "propella-4b"], "optional": ["allowed", "htmllang", "metalang"]}, "name": "mlt_Latn", "t/d": 1493.798459978408, "c/t": 2.5071673177587632, "samples": "https://data.hplt-project.org/three/sorted/mlt_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/mlt_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/mlt_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/mlt_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/mlt_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/mlt_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/mlt_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/mlt_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/mlt_Latn.md5"}
{"files": 6, "bytes": 27410407, "documents": 7844, "segments": 188572, "tokens": 17396885, "characters": 37089029, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["xml", "doc_scores", "s", "de", "o", "id", "openlid_v3", "filter", "ts", "u", "seg_langs_openlid_v3", "glotlid_v3", "f", "md", "c", "text", "crawl_id", "openlid_v2", "pii", "rs", "cluster_size"], "optional": ["allowed", "htmllang", "metalang"]}, "name": "mni_Beng", "t/d": 2217.8588730239676, "c/t": 2.131935056189657, "samples": "https://data.hplt-project.org/three/sorted/mni_Beng.json", "urls": ["https://data.hplt-project.org/three/sorted/mni_Beng/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/mni_Beng/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/mni_Beng/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/mni_Beng/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/mni_Beng/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/mni_Beng/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/mni_Beng.map", "md5": "https://data.hplt-project.org/three/sorted/mni_Beng.md5"}
{"files": 5, "bytes": 6192330, "documents": 2077, "segments": 52050, "tokens": 6244094, "characters": 12508389, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["doc_scores", "u", "text", "s", "pii", "openlid_v2", "de", "cluster_size", "f", "openlid_v3", "crawl_id", "c", "id", "md", "xml", "glotlid_v3", "filter", "ts", "rs", "o", "seg_langs_openlid_v3"], "optional": ["allowed", "htmllang", "metalang"]}, "name": "mos_Latn", "t/d": 3006.3042850264806, "c/t": 2.0032352171507988, "samples": "https://data.hplt-project.org/three/sorted/mos_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/mos_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/mos_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/mos_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/mos_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/mos_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/mos_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/mos_Latn.md5"}
{"files": 6, "bytes": 415553817, "documents": 230499, "segments": 5122363, "tokens": 282325403, "characters": 826411007, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["xml", "s", "de", "md", "id", "o", "ts", "crawl_id", "cluster_size", "pii", "openlid_v3", "openlid_v2", "f", "u", "filter", "glotlid_v3", "c", "text", "rs", "doc_scores", "seg_langs_openlid_v3"], "optional": ["htmllang", "metalang", "allowed"]}, "name": "mri_Latn", "t/d": 1224.8443724267784, "c/t": 2.927157805208198, "samples": "https://data.hplt-project.org/three/sorted/mri_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/mri_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/mri_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/mri_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/mri_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/mri_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/mri_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/mri_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/mri_Latn.md5"}
{"files": 6, "bytes": 6976970705, "documents": 2494038, "segments": 40281565, "tokens": 4703958061, "characters": 8336801005, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["u", "ts", "crawl_id", "c", "filter", "openlid_v2", "glotlid_v3", "xml", "web-register", "de", "cluster_size", "s", "seg_langs_openlid_v3", "o", "f", "openlid_v3", "text", "md", "rs", "doc_scores", "id", "pii"], "optional": ["allowed", "metalang", "htmllang"]}, "name": "mya_Mymr", "t/d": 1886.0811507282567, "c/t": 1.772294926291861, "samples": "https://data.hplt-project.org/three/sorted/mya_Mymr.json", "urls": ["https://data.hplt-project.org/three/sorted/mya_Mymr/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/mya_Mymr/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/mya_Mymr/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/mya_Mymr/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/mya_Mymr/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/mya_Mymr/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/mya_Mymr.map", "md5": "https://data.hplt-project.org/three/sorted/mya_Mymr.md5"}
{"files": 28, "bytes": 414782941014, "documents": 214553429, "segments": 4266955263, "tokens": 175457807310, "characters": 653189379416, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["text", "xml", "md", "f", "o", "s", "rs", "u", "c", "ts", "de", "id", "openlid_v2", "glotlid_v3", "openlid_v3", "seg_langs_openlid_v3", "cluster_size", "web-register", "crawl_id", "filter", "pii", "doc_scores", "bsc-edu", "finepdfs-edu", "fineweb2-hq", "jql", "propella-4b"], "optional": ["htmllang", "metalang", "allowed"]}, "name": "nld_Latn", "t/d": 817.7814175600987, "c/t": 3.7227718129518212, "samples": "https://data.hplt-project.org/three/sorted/nld_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/nld_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/nld_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/nld_Latn/5_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/nld_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/nld_Latn/6_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/nld_Latn/6_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/nld_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/nld_Latn/7_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/nld_Latn/7_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/nld_Latn/7_4.jsonl.zst", "https://data.hplt-project.org/three/sorted/nld_Latn/7_5.jsonl.zst", "https://data.hplt-project.org/three/sorted/nld_Latn/7_6.jsonl.zst", "https://data.hplt-project.org/three/sorted/nld_Latn/7_7.jsonl.zst", "https://data.hplt-project.org/three/sorted/nld_Latn/7_8.jsonl.zst", "https://data.hplt-project.org/three/sorted/nld_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/nld_Latn/8_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/nld_Latn/8_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/nld_Latn/8_4.jsonl.zst", "https://data.hplt-project.org/three/sorted/nld_Latn/8_5.jsonl.zst", "https://data.hplt-project.org/three/sorted/nld_Latn/8_6.jsonl.zst", "https://data.hplt-project.org/three/sorted/nld_Latn/8_7.jsonl.zst", "https://data.hplt-project.org/three/sorted/nld_Latn/8_8.jsonl.zst", "https://data.hplt-project.org/three/sorted/nld_Latn/8_9.jsonl.zst", "https://data.hplt-project.org/three/sorted/nld_Latn/9_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/nld_Latn/9_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/nld_Latn/9_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/nld_Latn/9_4.jsonl.zst", "https://data.hplt-project.org/three/sorted/nld_Latn/9_5.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/nld_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/nld_Latn.md5"}
{"files": 6, "bytes": 2835121641, "documents": 1478988, "segments": 27738602, "tokens": 1372529094, "characters": 4300528740, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["filter", "ts", "glotlid_v3", "de", "openlid_v3", "finepdfs-edu", "cluster_size", "s", "propella-4b", "crawl_id", "f", "md", "xml", "bsc-edu", "rs", "doc_scores", "o", "openlid_v2", "web-register", "jql", "pii", "id", "text", "u", "seg_langs_openlid_v3", "c"], "optional": ["allowed", "metalang", "htmllang"]}, "name": "nno_Latn", "t/d": 928.0190873759625, "c/t": 3.1332878543702476, "samples": "https://data.hplt-project.org/three/sorted/nno_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/nno_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/nno_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/nno_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/nno_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/nno_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/nno_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/nno_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/nno_Latn.md5"}
{"files": 8, "bytes": 82766658572, "documents": 37000822, "segments": 822024953, "tokens": 39691430615, "characters": 132759845015, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["text", "xml", "md", "f", "o", "s", "rs", "u", "c", "ts", "de", "id", "openlid_v2", "glotlid_v3", "openlid_v3", "seg_langs_openlid_v3", "cluster_size", "web-register", "crawl_id", "filter", "pii", "doc_scores", "bsc-edu", "finepdfs-edu", "jql", "propella-4b"], "optional": ["htmllang", "metalang", "allowed"]}, "name": "nob_Latn", "t/d": 1072.7175362482487, "c/t": 3.3447986872216195, "samples": "https://data.hplt-project.org/three/sorted/nob_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/nob_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/nob_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/nob_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/nob_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/nob_Latn/7_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/nob_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/nob_Latn/8_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/nob_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/nob_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/nob_Latn.md5"}
{"files": 6, "bytes": 16861419952, "documents": 9346034, "segments": 97354512, "tokens": 6309030989, "characters": 19650427398, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["text", "doc_scores", "f", "o", "ts", "openlid_v2", "pii", "xml", "crawl_id", "seg_langs_openlid_v3", "rs", "de", "c", "s", "md", "filter", "cluster_size", "openlid_v3", "u", "glotlid_v3", "web-register", "id"], "optional": ["allowed", "htmllang", "metalang"]}, "name": "npi_Deva", "t/d": 675.0490089165094, "c/t": 3.1146506384674852, "samples": "https://data.hplt-project.org/three/sorted/npi_Deva.json", "urls": ["https://data.hplt-project.org/three/sorted/npi_Deva/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/npi_Deva/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/npi_Deva/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/npi_Deva/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/npi_Deva/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/npi_Deva/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/npi_Deva.map", "md5": "https://data.hplt-project.org/three/sorted/npi_Deva.md5"}
{"files": 6, "bytes": 21269717, "documents": 8965, "segments": 251367, "tokens": 17007603, "characters": 45552224, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["filter", "u", "ts", "f", "seg_langs_openlid_v3", "crawl_id", "c", "rs", "text", "de", "openlid_v2", "md", "doc_scores", "openlid_v3", "id", "glotlid_v3", "o", "xml", "pii", "s", "cluster_size"], "optional": ["allowed", "metalang", "htmllang"]}, "name": "nso_Latn", "t/d": 1897.1113218070273, "c/t": 2.678344737938674, "samples": "https://data.hplt-project.org/three/sorted/nso_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/nso_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/nso_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/nso_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/nso_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/nso_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/nso_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/nso_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/nso_Latn.md5"}
{"files": 4, "bytes": 172207, "documents": 87, "segments": 3329, "tokens": 143678, "characters": 275079, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["crawl_id", "filter", "glotlid_v3", "c", "ts", "cluster_size", "o", "id", "openlid_v2", "f", "openlid_v3", "pii", "md", "de", "text", "seg_langs_openlid_v3", "s", "xml", "u", "rs", "doc_scores"], "optional": ["allowed", "metalang", "htmllang"]}, "name": "nus_Latn", "t/d": 1651.471264367816, "c/t": 1.9145519842982224, "samples": "https://data.hplt-project.org/three/sorted/nus_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/nus_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/nus_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/nus_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/nus_Latn/8_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/nus_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/nus_Latn.md5"}
{"files": 6, "bytes": 375090013, "documents": 200066, "segments": 4698067, "tokens": 256085103, "characters": 729918683, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["md", "s", "rs", "crawl_id", "openlid_v3", "de", "glotlid_v3", "xml", "o", "seg_langs_openlid_v3", "c", "doc_scores", "u", "filter", "id", "openlid_v2", "f", "cluster_size", "ts", "text", "pii"], "optional": ["metalang", "allowed", "htmllang"]}, "name": "nya_Latn", "t/d": 1280.0031139723892, "c/t": 2.8502973208871114, "samples": "https://data.hplt-project.org/three/sorted/nya_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/nya_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/nya_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/nya_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/nya_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/nya_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/nya_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/nya_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/nya_Latn.md5"}
{"files": 6, "bytes": 226695175, "documents": 123206, "segments": 2435274, "tokens": 124669509, "characters": 384290300, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["glotlid_v3", "filter", "md", "u", "openlid_v3", "xml", "de", "s", "pii", "id", "text", "crawl_id", "rs", "ts", "o", "c", "cluster_size", "f", "doc_scores", "seg_langs_openlid_v3", "openlid_v2"], "optional": ["htmllang", "metalang", "allowed"]}, "name": "oci_Latn", "t/d": 1011.8785529925491, "c/t": 3.0824722346504148, "samples": "https://data.hplt-project.org/three/sorted/oci_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/oci_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/oci_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/oci_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/oci_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/oci_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/oci_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/oci_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/oci_Latn.md5"}
{"files": 6, "bytes": 2806765592, "documents": 1985876, "segments": 12580428, "tokens": 2192972633, "characters": 3154711429, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["doc_scores", "filter", "rs", "u", "ts", "cluster_size", "glotlid_v3", "pii", "openlid_v2", "openlid_v3", "f", "web-register", "c", "s", "text", "md", "crawl_id", "seg_langs_openlid_v3", "id", "xml", "de", "o"], "optional": ["htmllang", "allowed", "metalang"]}, "name": "ory_Orya", "t/d": 1104.2847755851826, "c/t": 1.4385548554175687, "samples": "https://data.hplt-project.org/three/sorted/ory_Orya.json", "urls": ["https://data.hplt-project.org/three/sorted/ory_Orya/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ory_Orya/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ory_Orya/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ory_Orya/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ory_Orya/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ory_Orya/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/ory_Orya.map", "md5": "https://data.hplt-project.org/three/sorted/ory_Orya.md5"}
{"files": 6, "bytes": 7758568, "documents": 3478, "segments": 74804, "tokens": 5840397, "characters": 17266508, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["openlid_v3", "openlid_v2", "xml", "doc_scores", "id", "text", "c", "ts", "u", "s", "cluster_size", "pii", "filter", "seg_langs_openlid_v3", "o", "glotlid_v3", "f", "md", "crawl_id", "rs", "de"], "optional": ["metalang", "allowed", "htmllang"]}, "name": "pag_Latn", "t/d": 1679.2400805060379, "c/t": 2.9563928616496447, "samples": "https://data.hplt-project.org/three/sorted/pag_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/pag_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/pag_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/pag_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/pag_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/pag_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/pag_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/pag_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/pag_Latn.md5"}
{"files": 6, "bytes": 4126669447, "documents": 1770831, "segments": 24794472, "tokens": 2654045612, "characters": 4873433744, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["openlid_v2", "glotlid_v3", "s", "filter", "crawl_id", "pii", "cluster_size", "openlid_v3", "web-register", "c", "de", "u", "xml", "text", "ts", "f", "id", "seg_langs_openlid_v3", "o", "rs", "md", "doc_scores"], "optional": ["metalang", "allowed", "htmllang"]}, "name": "pan_Guru", "t/d": 1498.7571439623543, "c/t": 1.836228330803834, "samples": "https://data.hplt-project.org/three/sorted/pan_Guru.json", "urls": ["https://data.hplt-project.org/three/sorted/pan_Guru/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/pan_Guru/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/pan_Guru/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/pan_Guru/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/pan_Guru/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/pan_Guru/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/pan_Guru.map", "md5": "https://data.hplt-project.org/three/sorted/pan_Guru.md5"}
{"files": 6, "bytes": 253538635, "documents": 196618, "segments": 2551104, "tokens": 144973504, "characters": 493154302, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["pii", "o", "id", "c", "openlid_v2", "s", "ts", "de", "cluster_size", "seg_langs_openlid_v3", "md", "rs", "openlid_v3", "glotlid_v3", "doc_scores", "crawl_id", "text", "u", "filter", "f", "xml"], "optional": ["allowed", "htmllang", "metalang"]}, "name": "pap_Latn", "t/d": 737.3358695541608, "c/t": 3.4016857452793583, "samples": "https://data.hplt-project.org/three/sorted/pap_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/pap_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/pap_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/pap_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/pap_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/pap_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/pap_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/pap_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/pap_Latn.md5"}
{"files": 6, "bytes": 2063267956, "documents": 1086975, "segments": 17638671, "tokens": 1137911864, "characters": 2764286814, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["id", "pii", "s", "seg_langs_openlid_v3", "crawl_id", "glotlid_v3", "web-register", "openlid_v2", "openlid_v3", "u", "doc_scores", "text", "f", "xml", "de", "o", "rs", "filter", "md", "cluster_size", "c", "ts"], "optional": ["htmllang", "metalang", "allowed"]}, "name": "pbt_Arab", "t/d": 1046.86111824099, "c/t": 2.4292626709092824, "samples": "https://data.hplt-project.org/three/sorted/pbt_Arab.json", "urls": ["https://data.hplt-project.org/three/sorted/pbt_Arab/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/pbt_Arab/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/pbt_Arab/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/pbt_Arab/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/pbt_Arab/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/pbt_Arab/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/pbt_Arab.map", "md5": "https://data.hplt-project.org/three/sorted/pbt_Arab.md5"}
{"files": 6, "bytes": 770851861, "documents": 418312, "segments": 8018647, "tokens": 516377150, "characters": 1479221663, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["o", "id", "de", "rs", "pii", "cluster_size", "openlid_v3", "openlid_v2", "ts", "glotlid_v3", "md", "filter", "doc_scores", "crawl_id", "u", "xml", "text", "f", "seg_langs_openlid_v3", "web-register", "c", "s"], "optional": ["htmllang", "allowed", "metalang"]}, "name": "plt_Latn", "t/d": 1234.4306402876323, "c/t": 2.864614871126656, "samples": "https://data.hplt-project.org/three/sorted/plt_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/plt_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/plt_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/plt_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/plt_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/plt_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/plt_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/plt_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/plt_Latn.md5"}
{"files": 40, "bytes": 609434013092, "documents": 276726699, "segments": 5864286575, "tokens": 284147921352, "characters": 936416484131, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["text", "xml", "md", "f", "o", "s", "rs", "u", "c", "ts", "de", "id", "openlid_v2", "glotlid_v3", "openlid_v3", "seg_langs_openlid_v3", "cluster_size", "web-register", "crawl_id", "filter", "pii", "doc_scores", "bsc-edu", "finepdfs-edu", "fineweb2-hq", "jql", "propella-4b"], "optional": ["htmllang", "metalang", "allowed"]}, "name": "pol_Latn", "t/d": 1026.8178761890988, "c/t": 3.2955246678400836, "samples": "https://data.hplt-project.org/three/sorted/pol_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/pol_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/pol_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/pol_Latn/5_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/pol_Latn/5_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/pol_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/pol_Latn/6_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/pol_Latn/6_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/pol_Latn/6_4.jsonl.zst", "https://data.hplt-project.org/three/sorted/pol_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/pol_Latn/7_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/pol_Latn/7_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/pol_Latn/7_4.jsonl.zst", "https://data.hplt-project.org/three/sorted/pol_Latn/7_5.jsonl.zst", "https://data.hplt-project.org/three/sorted/pol_Latn/7_6.jsonl.zst", "https://data.hplt-project.org/three/sorted/pol_Latn/7_7.jsonl.zst", "https://data.hplt-project.org/three/sorted/pol_Latn/7_8.jsonl.zst", "https://data.hplt-project.org/three/sorted/pol_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/pol_Latn/8_10.jsonl.zst", "https://data.hplt-project.org/three/sorted/pol_Latn/8_11.jsonl.zst", "https://data.hplt-project.org/three/sorted/pol_Latn/8_12.jsonl.zst", "https://data.hplt-project.org/three/sorted/pol_Latn/8_13.jsonl.zst", "https://data.hplt-project.org/three/sorted/pol_Latn/8_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/pol_Latn/8_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/pol_Latn/8_4.jsonl.zst", "https://data.hplt-project.org/three/sorted/pol_Latn/8_5.jsonl.zst", "https://data.hplt-project.org/three/sorted/pol_Latn/8_6.jsonl.zst", "https://data.hplt-project.org/three/sorted/pol_Latn/8_7.jsonl.zst", "https://data.hplt-project.org/three/sorted/pol_Latn/8_8.jsonl.zst", "https://data.hplt-project.org/three/sorted/pol_Latn/8_9.jsonl.zst", "https://data.hplt-project.org/three/sorted/pol_Latn/9_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/pol_Latn/9_10.jsonl.zst", "https://data.hplt-project.org/three/sorted/pol_Latn/9_11.jsonl.zst", "https://data.hplt-project.org/three/sorted/pol_Latn/9_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/pol_Latn/9_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/pol_Latn/9_4.jsonl.zst", "https://data.hplt-project.org/three/sorted/pol_Latn/9_5.jsonl.zst", "https://data.hplt-project.org/three/sorted/pol_Latn/9_6.jsonl.zst", "https://data.hplt-project.org/three/sorted/pol_Latn/9_7.jsonl.zst", "https://data.hplt-project.org/three/sorted/pol_Latn/9_8.jsonl.zst", "https://data.hplt-project.org/three/sorted/pol_Latn/9_9.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/pol_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/pol_Latn.md5"}
{"files": 50, "bytes": 786928728106, "documents": 367761984, "segments": 8058047719, "tokens": 319877699038, "characters": 1256345215913, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["text", "xml", "md", "f", "o", "s", "rs", "u", "c", "ts", "de", "id", "openlid_v2", "glotlid_v3", "openlid_v3", "seg_langs_openlid_v3", "cluster_size", "web-register", "crawl_id", "filter", "pii", "doc_scores", "bsc-edu", "finepdfs-edu", "fineweb2-hq", "jql", "propella-4b"], "optional": ["htmllang", "metalang", "allowed"]}, "name": "por_Latn", "t/d": 869.7954463884989, "c/t": 3.9275798834721267, "samples": "https://data.hplt-project.org/three/sorted/por_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/por_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/por_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/por_Latn/5_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/por_Latn/5_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/por_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/por_Latn/6_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/por_Latn/6_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/por_Latn/6_4.jsonl.zst", "https://data.hplt-project.org/three/sorted/por_Latn/6_5.jsonl.zst", "https://data.hplt-project.org/three/sorted/por_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/por_Latn/7_10.jsonl.zst", "https://data.hplt-project.org/three/sorted/por_Latn/7_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/por_Latn/7_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/por_Latn/7_4.jsonl.zst", "https://data.hplt-project.org/three/sorted/por_Latn/7_5.jsonl.zst", "https://data.hplt-project.org/three/sorted/por_Latn/7_6.jsonl.zst", "https://data.hplt-project.org/three/sorted/por_Latn/7_7.jsonl.zst", "https://data.hplt-project.org/three/sorted/por_Latn/7_8.jsonl.zst", "https://data.hplt-project.org/three/sorted/por_Latn/7_9.jsonl.zst", "https://data.hplt-project.org/three/sorted/por_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/por_Latn/8_10.jsonl.zst", "https://data.hplt-project.org/three/sorted/por_Latn/8_11.jsonl.zst", "https://data.hplt-project.org/three/sorted/por_Latn/8_12.jsonl.zst", "https://data.hplt-project.org/three/sorted/por_Latn/8_13.jsonl.zst", "https://data.hplt-project.org/three/sorted/por_Latn/8_14.jsonl.zst", "https://data.hplt-project.org/three/sorted/por_Latn/8_15.jsonl.zst", "https://data.hplt-project.org/three/sorted/por_Latn/8_16.jsonl.zst", "https://data.hplt-project.org/three/sorted/por_Latn/8_17.jsonl.zst", "https://data.hplt-project.org/three/sorted/por_Latn/8_18.jsonl.zst", "https://data.hplt-project.org/three/sorted/por_Latn/8_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/por_Latn/8_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/por_Latn/8_4.jsonl.zst", "https://data.hplt-project.org/three/sorted/por_Latn/8_5.jsonl.zst", "https://data.hplt-project.org/three/sorted/por_Latn/8_6.jsonl.zst", "https://data.hplt-project.org/three/sorted/por_Latn/8_7.jsonl.zst", "https://data.hplt-project.org/three/sorted/por_Latn/8_8.jsonl.zst", "https://data.hplt-project.org/three/sorted/por_Latn/8_9.jsonl.zst", "https://data.hplt-project.org/three/sorted/por_Latn/9_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/por_Latn/9_10.jsonl.zst", "https://data.hplt-project.org/three/sorted/por_Latn/9_11.jsonl.zst", "https://data.hplt-project.org/three/sorted/por_Latn/9_12.jsonl.zst", "https://data.hplt-project.org/three/sorted/por_Latn/9_13.jsonl.zst", "https://data.hplt-project.org/three/sorted/por_Latn/9_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/por_Latn/9_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/por_Latn/9_4.jsonl.zst", "https://data.hplt-project.org/three/sorted/por_Latn/9_5.jsonl.zst", "https://data.hplt-project.org/three/sorted/por_Latn/9_6.jsonl.zst", "https://data.hplt-project.org/three/sorted/por_Latn/9_7.jsonl.zst", "https://data.hplt-project.org/three/sorted/por_Latn/9_8.jsonl.zst", "https://data.hplt-project.org/three/sorted/por_Latn/9_9.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/por_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/por_Latn.md5"}
{"files": 6, "bytes": 10727886, "documents": 6914, "segments": 116211, "tokens": 7489449, "characters": 20065220, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["glotlid_v3", "text", "filter", "xml", "ts", "id", "o", "f", "cluster_size", "seg_langs_openlid_v3", "md", "u", "rs", "openlid_v3", "pii", "s", "de", "crawl_id", "doc_scores", "openlid_v2", "c"], "optional": ["allowed", "metalang", "htmllang"]}, "name": "quy_Latn", "t/d": 1083.2295342782759, "c/t": 2.679131669098755, "samples": "https://data.hplt-project.org/three/sorted/quy_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/quy_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/quy_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/quy_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/quy_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/quy_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/quy_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/quy_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/quy_Latn.md5"}
{"files": 16, "bytes": 202226508443, "documents": 101192506, "segments": 1932750569, "tokens": 92305335246, "characters": 307184059221, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["text", "xml", "md", "f", "o", "s", "rs", "u", "c", "ts", "de", "id", "openlid_v2", "glotlid_v3", "openlid_v3", "seg_langs_openlid_v3", "cluster_size", "web-register", "crawl_id", "filter", "pii", "doc_scores", "bsc-edu", "finepdfs-edu", "jql", "propella-4b"], "optional": ["htmllang", "metalang", "allowed"]}, "name": "ron_Latn", "t/d": 912.1756036558676, "c/t": 3.3279122859186154, "samples": "https://data.hplt-project.org/three/sorted/ron_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/ron_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ron_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ron_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ron_Latn/6_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/ron_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ron_Latn/7_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/ron_Latn/7_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/ron_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ron_Latn/8_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/ron_Latn/8_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/ron_Latn/8_4.jsonl.zst", "https://data.hplt-project.org/three/sorted/ron_Latn/8_5.jsonl.zst", "https://data.hplt-project.org/three/sorted/ron_Latn/9_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ron_Latn/9_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/ron_Latn/9_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/ron_Latn/9_4.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/ron_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/ron_Latn.md5"}
{"files": 6, "bytes": 84451926, "documents": 73048, "segments": 847450, "tokens": 57979909, "characters": 158703534, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["filter", "crawl_id", "ts", "cluster_size", "openlid_v2", "pii", "s", "glotlid_v3", "de", "seg_langs_openlid_v3", "id", "c", "openlid_v3", "o", "xml", "f", "text", "doc_scores", "rs", "u", "md"], "optional": ["metalang", "htmllang", "allowed"]}, "name": "run_Latn", "t/d": 793.723428430621, "c/t": 2.737215989766386, "samples": "https://data.hplt-project.org/three/sorted/run_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/run_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/run_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/run_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/run_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/run_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/run_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/run_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/run_Latn.md5"}
{"files": 487, "bytes": 6186520620839, "documents": 1679796206, "segments": 54931094395, "tokens": 2413715300308, "characters": 8657101439347, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["text", "xml", "md", "f", "o", "s", "rs", "u", "c", "ts", "de", "id", "openlid_v2", "glotlid_v3", "openlid_v3", "seg_langs_openlid_v3", "cluster_size", "web-register", "crawl_id", "filter", "pii", "doc_scores"], "optional": ["htmllang", "metalang"]}, "name": "rus_Cyrl", "t/d": 1436.9096034903177, "c/t": 3.586629060288228, "samples": "https://data.hplt-project.org/three/sorted/rus_Cyrl.json", "urls": ["https://data.hplt-project.org/three/sorted/rus_Cyrl/10_1.ab.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/10_1.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/10_1.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/10_2.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/10_3.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/5_1.ab.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/5_1.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/5_1.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/5_10.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/5_11.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/5_12.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/5_13.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/5_14.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/5_15.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/5_16.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/5_17.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/5_2.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/5_2.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/5_3.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/5_3.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/5_4.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/5_4.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/5_5.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/5_5.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/5_6.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/5_7.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/5_8.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/5_9.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/6_1.ab.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/6_1.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/6_1.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/6_10.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/6_11.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/6_12.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/6_13.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/6_14.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/6_15.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/6_16.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/6_17.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/6_18.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/6_19.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/6_2.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/6_2.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/6_20.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/6_21.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/6_22.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/6_23.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/6_24.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/6_25.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/6_26.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/6_27.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/6_28.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/6_3.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/6_3.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/6_4.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/6_4.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/6_5.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/6_5.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/6_6.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/6_6.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/6_7.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/6_7.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/6_8.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/6_8.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/6_9.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/7_1.ab.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/7_1.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/7_1.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/7_10.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/7_10.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/7_11.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/7_11.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/7_12.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/7_12.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/7_13.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/7_13.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/7_14.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/7_15.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/7_16.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/7_17.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/7_18.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/7_19.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/7_2.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/7_2.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/7_20.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/7_21.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/7_22.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/7_23.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/7_24.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/7_25.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/7_26.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/7_27.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/7_28.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/7_29.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/7_3.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/7_3.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/7_30.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/7_31.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/7_32.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/7_33.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/7_34.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/7_35.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/7_36.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/7_37.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/7_38.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/7_39.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/7_4.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/7_4.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/7_40.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/7_41.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/7_42.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/7_43.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/7_44.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/7_45.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/7_46.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/7_47.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/7_48.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/7_49.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/7_5.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/7_5.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/7_50.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/7_51.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/7_52.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/7_53.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/7_54.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/7_55.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/7_56.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/7_57.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/7_58.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/7_59.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/7_6.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/7_6.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/7_60.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/7_7.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/7_7.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/7_8.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/7_8.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/7_9.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/7_9.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_1.ab.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_1.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_1.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_10.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_10.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_100.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_101.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_102.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_103.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_104.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_105.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_106.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_107.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_108.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_109.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_11.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_11.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_110.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_111.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_112.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_113.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_114.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_115.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_116.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_117.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_118.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_119.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_12.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_12.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_120.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_121.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_122.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_123.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_124.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_125.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_126.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_127.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_128.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_129.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_13.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_13.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_130.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_131.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_132.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_14.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_14.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_15.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_15.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_16.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_16.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_17.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_17.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_18.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_18.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_19.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_2.ab.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_2.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_2.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_20.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_21.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_22.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_23.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_24.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_25.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_26.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_27.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_28.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_29.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_3.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_3.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_30.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_31.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_32.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_33.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_34.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_35.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_36.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_37.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_38.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_39.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_4.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_4.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_40.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_41.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_42.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_43.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_44.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_45.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_46.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_47.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_48.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_49.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_5.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_5.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_50.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_51.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_52.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_53.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_54.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_55.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_56.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_57.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_58.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_59.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_6.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_6.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_60.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_61.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_62.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_63.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_64.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_65.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_66.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_67.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_68.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_69.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_7.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_7.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_70.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_71.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_72.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_73.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_74.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_75.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_76.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_77.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_78.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_79.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_8.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_8.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_80.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_81.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_82.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_83.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_84.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_85.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_86.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_87.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_88.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_89.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_9.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_9.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_90.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_91.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_92.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_93.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_94.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_95.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_96.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_97.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_98.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/8_99.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_1.ab.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_1.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_1.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_10.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_10.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_100.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_101.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_102.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_103.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_104.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_105.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_106.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_107.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_108.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_109.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_11.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_11.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_110.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_111.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_112.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_113.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_114.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_115.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_116.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_117.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_118.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_119.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_12.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_12.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_120.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_121.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_122.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_123.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_124.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_125.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_126.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_127.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_128.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_129.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_13.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_13.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_130.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_131.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_132.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_133.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_134.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_135.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_136.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_137.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_138.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_139.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_14.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_14.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_140.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_141.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_142.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_143.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_144.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_145.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_146.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_147.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_148.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_149.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_15.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_15.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_150.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_151.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_152.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_153.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_154.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_155.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_156.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_157.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_158.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_159.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_16.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_16.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_160.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_161.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_162.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_163.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_164.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_165.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_166.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_167.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_168.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_169.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_17.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_17.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_170.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_171.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_172.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_18.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_18.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_19.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_19.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_2.ab.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_2.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_2.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_20.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_20.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_21.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_21.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_22.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_23.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_24.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_25.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_26.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_27.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_28.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_29.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_3.ab.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_3.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_3.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_30.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_31.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_32.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_33.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_34.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_35.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_36.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_37.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_38.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_39.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_4.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_4.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_40.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_41.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_42.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_43.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_44.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_45.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_46.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_47.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_48.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_49.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_5.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_5.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_50.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_51.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_52.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_53.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_54.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_55.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_56.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_57.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_58.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_59.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_6.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_6.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_60.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_61.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_62.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_63.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_64.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_65.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_66.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_67.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_68.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_69.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_7.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_7.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_70.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_71.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_72.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_73.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_74.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_75.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_76.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_77.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_78.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_79.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_8.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_8.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_80.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_81.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_82.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_83.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_84.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_85.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_86.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_87.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_88.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_89.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_9.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_9.ia.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_90.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_91.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_92.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_93.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_94.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_95.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_96.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_97.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_98.cc.jsonl.zst", "https://data.hplt-project.org/three/sorted/rus_Cyrl/9_99.cc.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/rus_Cyrl.map", "md5": "https://data.hplt-project.org/three/sorted/rus_Cyrl.md5"}
{"files": 5, "bytes": 5956655, "documents": 2609, "segments": 58131, "tokens": 5085561, "characters": 13932380, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["filter", "text", "o", "crawl_id", "openlid_v3", "ts", "pii", "openlid_v2", "c", "s", "glotlid_v3", "de", "rs", "f", "md", "doc_scores", "seg_langs_openlid_v3", "xml", "id", "cluster_size", "u"], "optional": ["htmllang", "metalang", "allowed"]}, "name": "sag_Latn", "t/d": 1949.2376389421233, "c/t": 2.7395954939877822, "samples": "https://data.hplt-project.org/three/sorted/sag_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/sag_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/sag_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/sag_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/sag_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/sag_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/sag_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/sag_Latn.md5"}
{"files": 6, "bytes": 654565194, "documents": 88208, "segments": 6019454, "tokens": 265614916, "characters": 613490164, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["id", "md", "crawl_id", "de", "rs", "ts", "glotlid_v3", "seg_langs_openlid_v3", "text", "pii", "f", "u", "doc_scores", "o", "web-register", "openlid_v2", "filter", "s", "c", "xml", "cluster_size", "openlid_v3"], "optional": ["allowed", "metalang", "htmllang"]}, "name": "san_Deva", "t/d": 3011.233856339561, "c/t": 2.3096977129100686, "samples": "https://data.hplt-project.org/three/sorted/san_Deva.json", "urls": ["https://data.hplt-project.org/three/sorted/san_Deva/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/san_Deva/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/san_Deva/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/san_Deva/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/san_Deva/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/san_Deva/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/san_Deva.map", "md5": "https://data.hplt-project.org/three/sorted/san_Deva.md5"}
{"files": 5, "bytes": 14828500, "documents": 6811, "segments": 105420, "tokens": 16363948, "characters": 17419815, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["md", "doc_scores", "id", "u", "de", "o", "c", "text", "rs", "ts", "pii", "openlid_v2", "s", "openlid_v3", "f", "filter", "seg_langs_openlid_v3", "xml", "cluster_size", "glotlid_v3", "crawl_id"], "optional": ["allowed", "htmllang", "metalang"]}, "name": "sat_Olck", "t/d": 2402.576420496256, "c/t": 1.0645239767322654, "samples": "https://data.hplt-project.org/three/sorted/sat_Olck.json", "urls": ["https://data.hplt-project.org/three/sorted/sat_Olck/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/sat_Olck/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/sat_Olck/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/sat_Olck/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/sat_Olck/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/sat_Olck.map", "md5": "https://data.hplt-project.org/three/sorted/sat_Olck.md5"}
{"files": 6, "bytes": 29806905, "documents": 13543, "segments": 467235, "tokens": 16536599, "characters": 46600855, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["rs", "xml", "f", "doc_scores", "filter", "pii", "c", "text", "crawl_id", "ts", "openlid_v2", "glotlid_v3", "openlid_v3", "o", "u", "s", "seg_langs_openlid_v3", "cluster_size", "de", "md", "id"], "optional": ["metalang", "htmllang", "allowed"]}, "name": "scn_Latn", "t/d": 1221.0440079745995, "c/t": 2.8180434804036794, "samples": "https://data.hplt-project.org/three/sorted/scn_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/scn_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/scn_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/scn_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/scn_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/scn_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/scn_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/scn_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/scn_Latn.md5"}
{"files": 5, "bytes": 35698034, "documents": 16778, "segments": 192975, "tokens": 38182055, "characters": 50010301, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["seg_langs_openlid_v3", "u", "s", "id", "o", "c", "openlid_v3", "doc_scores", "md", "text", "rs", "pii", "glotlid_v3", "filter", "ts", "openlid_v2", "cluster_size", "crawl_id", "de", "f", "xml"], "optional": ["metalang", "allowed", "htmllang"]}, "name": "shn_Mymr", "t/d": 2275.721480510192, "c/t": 1.309785473830573, "samples": "https://data.hplt-project.org/three/sorted/shn_Mymr.json", "urls": ["https://data.hplt-project.org/three/sorted/shn_Mymr/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/shn_Mymr/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/shn_Mymr/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/shn_Mymr/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/shn_Mymr/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/shn_Mymr.map", "md5": "https://data.hplt-project.org/three/sorted/shn_Mymr.md5"}
{"files": 6, "bytes": 4727031051, "documents": 1526030, "segments": 35416167, "tokens": 2692820779, "characters": 5495250628, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["de", "openlid_v2", "seg_langs_openlid_v3", "rs", "glotlid_v3", "md", "web-register", "openlid_v3", "xml", "crawl_id", "text", "id", "u", "cluster_size", "ts", "s", "doc_scores", "c", "f", "filter", "pii", "o"], "optional": ["metalang", "allowed", "htmllang"]}, "name": "sin_Sinh", "t/d": 1764.592294384776, "c/t": 2.040704183083697, "samples": "https://data.hplt-project.org/three/sorted/sin_Sinh.json", "urls": ["https://data.hplt-project.org/three/sorted/sin_Sinh/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/sin_Sinh/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/sin_Sinh/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/sin_Sinh/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/sin_Sinh/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/sin_Sinh/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/sin_Sinh.map", "md5": "https://data.hplt-project.org/three/sorted/sin_Sinh.md5"}
{"files": 9, "bytes": 86100815804, "documents": 39490021, "segments": 806860916, "tokens": 42422521166, "characters": 122745939612, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["text", "xml", "md", "f", "o", "s", "rs", "u", "c", "ts", "de", "id", "openlid_v2", "glotlid_v3", "openlid_v3", "seg_langs_openlid_v3", "cluster_size", "web-register", "crawl_id", "filter", "pii", "doc_scores", "bsc-edu", "finepdfs-edu", "jql", "propella-4b"], "optional": ["htmllang", "metalang", "allowed"]}, "name": "slk_Latn", "t/d": 1074.2592708674426, "c/t": 2.8934145411040797, "samples": "https://data.hplt-project.org/three/sorted/slk_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/slk_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/slk_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/slk_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/slk_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/slk_Latn/7_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/slk_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/slk_Latn/8_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/slk_Latn/9_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/slk_Latn/9_2.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/slk_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/slk_Latn.md5"}
{"files": 6, "bytes": 41682036512, "documents": 18234487, "segments": 407757315, "tokens": 21537422961, "characters": 64320086917, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["text", "xml", "md", "f", "o", "s", "rs", "u", "c", "ts", "de", "id", "openlid_v2", "glotlid_v3", "openlid_v3", "seg_langs_openlid_v3", "cluster_size", "web-register", "crawl_id", "filter", "pii", "doc_scores", "bsc-edu", "finepdfs-edu", "jql", "propella-4b"], "optional": ["htmllang", "metalang", "allowed"]}, "name": "slv_Latn", "t/d": 1181.1367635952686, "c/t": 2.9864337545615793, "samples": "https://data.hplt-project.org/three/sorted/slv_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/slv_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/slv_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/slv_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/slv_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/slv_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/slv_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/slv_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/slv_Latn.md5"}
{"files": 6, "bytes": 347475848, "documents": 186965, "segments": 3963083, "tokens": 263392240, "characters": 696870523, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["crawl_id", "openlid_v3", "f", "pii", "xml", "c", "text", "doc_scores", "s", "cluster_size", "openlid_v2", "ts", "seg_langs_openlid_v3", "glotlid_v3", "id", "filter", "u", "de", "o", "md", "rs"], "optional": ["htmllang", "metalang", "allowed"]}, "name": "smo_Latn", "t/d": 1408.7783274944509, "c/t": 2.6457519135719414, "samples": "https://data.hplt-project.org/three/sorted/smo_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/smo_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/smo_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/smo_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/smo_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/smo_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/smo_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/smo_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/smo_Latn.md5"}
{"files": 6, "bytes": 395025278, "documents": 212214, "segments": 4492429, "tokens": 258396052, "characters": 748208286, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["c", "text", "filter", "md", "f", "s", "o", "seg_langs_openlid_v3", "id", "cluster_size", "pii", "u", "crawl_id", "xml", "de", "rs", "openlid_v2", "glotlid_v3", "openlid_v3", "doc_scores", "ts"], "optional": ["allowed", "metalang", "htmllang"]}, "name": "sna_Latn", "t/d": 1217.6201947091145, "c/t": 2.8955871431038736, "samples": "https://data.hplt-project.org/three/sorted/sna_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/sna_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/sna_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/sna_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/sna_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/sna_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/sna_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/sna_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/sna_Latn.md5"}
{"files": 6, "bytes": 942336998, "documents": 420992, "segments": 7288518, "tokens": 586112186, "characters": 1290002809, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["crawl_id", "de", "id", "openlid_v2", "u", "md", "o", "openlid_v3", "web-register", "doc_scores", "seg_langs_openlid_v3", "ts", "c", "text", "f", "cluster_size", "filter", "s", "xml", "rs", "pii", "glotlid_v3"], "optional": ["metalang", "htmllang", "allowed"]}, "name": "snd_Arab", "t/d": 1392.216920986622, "c/t": 2.200948623511472, "samples": "https://data.hplt-project.org/three/sorted/snd_Arab.json", "urls": ["https://data.hplt-project.org/three/sorted/snd_Arab/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/snd_Arab/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/snd_Arab/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/snd_Arab/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/snd_Arab/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/snd_Arab/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/snd_Arab.map", "md5": "https://data.hplt-project.org/three/sorted/snd_Arab.md5"}
{"files": 6, "bytes": 2024827696, "documents": 1458437, "segments": 20288646, "tokens": 1226656467, "characters": 3511132691, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["openlid_v3", "de", "filter", "rs", "f", "ts", "o", "seg_langs_openlid_v3", "c", "id", "glotlid_v3", "openlid_v2", "xml", "s", "text", "cluster_size", "pii", "u", "doc_scores", "md", "web-register", "crawl_id"], "optional": ["metalang", "htmllang", "allowed"]}, "name": "som_Latn", "t/d": 841.0760745921833, "c/t": 2.8623602332502114, "samples": "https://data.hplt-project.org/three/sorted/som_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/som_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/som_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/som_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/som_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/som_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/som_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/som_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/som_Latn.md5"}
{"files": 6, "bytes": 347259114, "documents": 174766, "segments": 4099042, "tokens": 245126009, "characters": 692399148, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["xml", "seg_langs_openlid_v3", "ts", "text", "cluster_size", "openlid_v2", "openlid_v3", "pii", "rs", "c", "de", "f", "s", "u", "filter", "doc_scores", "glotlid_v3", "id", "o", "crawl_id", "md"], "optional": ["metalang", "htmllang", "allowed"]}, "name": "sot_Latn", "t/d": 1402.5955220122908, "c/t": 2.8246661822001924, "samples": "https://data.hplt-project.org/three/sorted/sot_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/sot_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/sot_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/sot_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/sot_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/sot_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/sot_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/sot_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/sot_Latn.md5"}
{"files": 160, "bytes": 2610495473071, "documents": 1117429998, "segments": 24899826798, "tokens": 1013710912097, "characters": 4260131280485, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["fineweb2-hq", "md", "f", "doc_scores", "finepdfs-edu", "xml", "cluster_size", "glotlid_v3", "openlid_v3", "crawl_id", "u", "id", "ts", "propella-4b", "c", "rs", "bsc-edu", "text", "seg_langs_openlid_v3", "openlid_v2", "pii", "filter", "s", "o", "de", "web-register", "jql"], "optional": ["allowed", "metalang", "htmllang"]}, "name": "spa_Latn", "t/d": 907.1806859591754, "c/t": 4.202511021285481, "samples": "https://data.hplt-project.org/three/sorted/spa_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/spa_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/5_10.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/5_11.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/5_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/5_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/5_4.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/5_5.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/5_6.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/5_7.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/5_8.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/5_9.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/6_10.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/6_11.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/6_12.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/6_13.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/6_14.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/6_15.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/6_16.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/6_17.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/6_18.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/6_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/6_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/6_4.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/6_5.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/6_6.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/6_7.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/6_8.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/6_9.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/7_10.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/7_11.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/7_12.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/7_13.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/7_14.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/7_15.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/7_16.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/7_17.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/7_18.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/7_19.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/7_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/7_20.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/7_21.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/7_22.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/7_23.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/7_24.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/7_25.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/7_26.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/7_27.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/7_28.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/7_29.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/7_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/7_30.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/7_31.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/7_32.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/7_33.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/7_34.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/7_35.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/7_36.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/7_37.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/7_38.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/7_39.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/7_4.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/7_40.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/7_5.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/7_6.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/7_7.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/7_8.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/7_9.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/8_10.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/8_11.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/8_12.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/8_13.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/8_14.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/8_15.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/8_16.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/8_17.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/8_18.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/8_19.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/8_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/8_20.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/8_21.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/8_22.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/8_23.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/8_24.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/8_25.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/8_26.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/8_27.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/8_28.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/8_29.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/8_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/8_30.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/8_31.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/8_32.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/8_33.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/8_34.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/8_35.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/8_36.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/8_37.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/8_38.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/8_39.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/8_4.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/8_40.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/8_41.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/8_42.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/8_43.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/8_44.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/8_45.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/8_46.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/8_47.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/8_48.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/8_49.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/8_5.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/8_50.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/8_51.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/8_52.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/8_53.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/8_54.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/8_55.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/8_56.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/8_57.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/8_58.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/8_59.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/8_6.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/8_60.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/8_7.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/8_8.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/8_9.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/9_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/9_10.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/9_11.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/9_12.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/9_13.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/9_14.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/9_15.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/9_16.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/9_17.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/9_18.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/9_19.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/9_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/9_20.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/9_21.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/9_22.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/9_23.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/9_24.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/9_25.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/9_26.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/9_27.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/9_28.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/9_29.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/9_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/9_30.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/9_4.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/9_5.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/9_6.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/9_7.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/9_8.jsonl.zst", "https://data.hplt-project.org/three/sorted/spa_Latn/9_9.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/spa_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/spa_Latn.md5"}
{"files": 6, "bytes": 99498627, "documents": 64216, "segments": 756687, "tokens": 55265664, "characters": 165386662, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["f", "id", "md", "xml", "openlid_v3", "openlid_v2", "seg_langs_openlid_v3", "o", "doc_scores", "text", "cluster_size", "filter", "crawl_id", "c", "u", "rs", "glotlid_v3", "de", "pii", "ts", "s"], "optional": ["htmllang", "allowed", "metalang"]}, "name": "srd_Latn", "t/d": 860.6214027656658, "c/t": 2.9925753176511187, "samples": "https://data.hplt-project.org/three/sorted/srd_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/srd_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/srd_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/srd_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/srd_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/srd_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/srd_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/srd_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/srd_Latn.md5"}
{"files": 6, "bytes": 21662182939, "documents": 7566167, "segments": 174809153, "tokens": 10859427368, "characters": 28978416505, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["text", "xml", "md", "f", "o", "s", "rs", "u", "c", "ts", "de", "id", "openlid_v2", "glotlid_v3", "openlid_v3", "seg_langs_openlid_v3", "cluster_size", "crawl_id", "filter", "pii", "doc_scores", "bsc-edu", "finepdfs-edu", "jql", "propella-4b"], "optional": ["allowed", "htmllang", "metalang"]}, "name": "srp_Cyrl", "t/d": 1435.2613903446752, "c/t": 2.6685031837306727, "samples": "https://data.hplt-project.org/three/sorted/srp_Cyrl.json", "urls": ["https://data.hplt-project.org/three/sorted/srp_Cyrl/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/srp_Cyrl/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/srp_Cyrl/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/srp_Cyrl/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/srp_Cyrl/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/srp_Cyrl/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/srp_Cyrl.map", "md5": "https://data.hplt-project.org/three/sorted/srp_Cyrl.md5"}
{"files": 6, "bytes": 2270267315, "documents": 1340747, "segments": 20473619, "tokens": 1188539909, "characters": 3692166047, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["text", "xml", "md", "f", "o", "s", "rs", "u", "c", "ts", "de", "id", "openlid_v2", "glotlid_v3", "openlid_v3", "seg_langs_openlid_v3", "cluster_size", "crawl_id", "filter", "pii", "doc_scores"], "optional": ["htmllang", "metalang", "allowed"]}, "name": "srp_Latn", "t/d": 886.4759041042046, "c/t": 3.1064720831347366, "samples": "https://data.hplt-project.org/three/sorted/srp_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/srp_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/srp_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/srp_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/srp_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/srp_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/srp_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/srp_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/srp_Latn.md5"}
{"files": 6, "bytes": 8579225, "documents": 3133, "segments": 100586, "tokens": 6810864, "characters": 18246544, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["text", "de", "id", "doc_scores", "crawl_id", "pii", "openlid_v3", "filter", "o", "glotlid_v3", "xml", "seg_langs_openlid_v3", "f", "u", "openlid_v2", "md", "cluster_size", "s", "rs", "ts", "c"], "optional": ["allowed", "htmllang", "metalang"]}, "name": "ssw_Latn", "t/d": 2173.9112671560806, "c/t": 2.6790351415033395, "samples": "https://data.hplt-project.org/three/sorted/ssw_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/ssw_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ssw_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ssw_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ssw_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ssw_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ssw_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/ssw_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/ssw_Latn.md5"}
{"files": 6, "bytes": 494236103, "documents": 236153, "segments": 5728878, "tokens": 270727145, "characters": 881481903, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["openlid_v3", "u", "rs", "cluster_size", "text", "crawl_id", "de", "xml", "pii", "c", "web-register", "filter", "ts", "glotlid_v3", "md", "doc_scores", "seg_langs_openlid_v3", "id", "s", "openlid_v2", "f", "o"], "optional": ["htmllang", "allowed", "metalang"]}, "name": "sun_Latn", "t/d": 1146.405698847781, "c/t": 3.2559790153292534, "samples": "https://data.hplt-project.org/three/sorted/sun_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/sun_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/sun_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/sun_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/sun_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/sun_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/sun_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/sun_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/sun_Latn.md5"}
{"files": 18, "bytes": 229943236853, "documents": 106942233, "segments": 2432647075, "tokens": 107849625417, "characters": 362684163364, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["text", "xml", "md", "f", "o", "s", "rs", "u", "c", "ts", "de", "id", "openlid_v2", "glotlid_v3", "openlid_v3", "seg_langs_openlid_v3", "cluster_size", "web-register", "crawl_id", "filter", "pii", "doc_scores", "bsc-edu", "finepdfs-edu", "fineweb2-hq", "jql", "propella-4b"], "optional": ["htmllang", "metalang", "allowed"]}, "name": "swe_Latn", "t/d": 1008.4848837689783, "c/t": 3.3628690128656786, "samples": "https://data.hplt-project.org/three/sorted/swe_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/swe_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/swe_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/swe_Latn/5_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/swe_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/swe_Latn/6_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/swe_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/swe_Latn/7_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/swe_Latn/7_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/swe_Latn/7_4.jsonl.zst", "https://data.hplt-project.org/three/sorted/swe_Latn/7_5.jsonl.zst", "https://data.hplt-project.org/three/sorted/swe_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/swe_Latn/8_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/swe_Latn/8_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/swe_Latn/8_4.jsonl.zst", "https://data.hplt-project.org/three/sorted/swe_Latn/8_5.jsonl.zst", "https://data.hplt-project.org/three/sorted/swe_Latn/9_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/swe_Latn/9_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/swe_Latn/9_3.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/swe_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/swe_Latn.md5"}
{"files": 6, "bytes": 4115528825, "documents": 2299928, "segments": 54800759, "tokens": 2480896357, "characters": 7442358872, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["ts", "pii", "openlid_v3", "de", "rs", "s", "doc_scores", "u", "c", "glotlid_v3", "xml", "filter", "crawl_id", "f", "seg_langs_openlid_v3", "cluster_size", "web-register", "openlid_v2", "o", "id", "text", "md"], "optional": ["metalang", "allowed", "htmllang"]}, "name": "swh_Latn", "t/d": 1078.6843575103221, "c/t": 2.9998669033476273, "samples": "https://data.hplt-project.org/three/sorted/swh_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/swh_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/swh_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/swh_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/swh_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/swh_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/swh_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/swh_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/swh_Latn.md5"}
{"files": 6, "bytes": 22866277, "documents": 13616, "segments": 225026, "tokens": 14652379, "characters": 38304484, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["u", "id", "crawl_id", "de", "glotlid_v3", "s", "filter", "pii", "xml", "openlid_v2", "openlid_v3", "f", "seg_langs_openlid_v3", "doc_scores", "cluster_size", "o", "c", "md", "text", "ts", "rs"], "optional": ["metalang", "htmllang", "allowed"]}, "name": "szl_Latn", "t/d": 1076.1147914218566, "c/t": 2.614216025943637, "samples": "https://data.hplt-project.org/three/sorted/szl_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/szl_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/szl_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/szl_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/szl_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/szl_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/szl_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/szl_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/szl_Latn.md5"}
{"files": 7, "bytes": 27517626100, "documents": 11701837, "segments": 200471377, "tokens": 8928059678, "characters": 32426396917, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["openlid_v2", "seg_langs_openlid_v3", "md", "de", "c", "ts", "u", "o", "pii", "xml", "crawl_id", "id", "s", "f", "filter", "web-register", "doc_scores", "text", "cluster_size", "glotlid_v3", "openlid_v3", "rs"], "optional": ["htmllang", "allowed", "metalang"]}, "name": "tam_Taml", "t/d": 762.9622321691885, "c/t": 3.6319646246208706, "samples": "https://data.hplt-project.org/three/sorted/tam_Taml.json", "urls": ["https://data.hplt-project.org/three/sorted/tam_Taml/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/tam_Taml/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/tam_Taml/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/tam_Taml/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/tam_Taml/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/tam_Taml/8_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/tam_Taml/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/tam_Taml.map", "md5": "https://data.hplt-project.org/three/sorted/tam_Taml.md5"}
{"files": 5, "bytes": 1759774, "documents": 350, "segments": 21222, "tokens": 2142620, "characters": 3727782, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["id", "glotlid_v3", "de", "o", "crawl_id", "seg_langs_openlid_v3", "c", "text", "md", "xml", "f", "openlid_v2", "u", "doc_scores", "s", "filter", "pii", "cluster_size", "openlid_v3", "rs", "ts"], "optional": ["allowed", "htmllang", "metalang"]}, "name": "taq_Latn", "t/d": 6121.771428571428, "c/t": 1.7398241405382195, "samples": "https://data.hplt-project.org/three/sorted/taq_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/taq_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/taq_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/taq_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/taq_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/taq_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/taq_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/taq_Latn.md5"}
{"files": 3, "bytes": 27990, "documents": 5, "segments": 143, "tokens": 38922, "characters": 36688, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["doc_scores", "seg_langs_openlid_v3", "openlid_v3", "u", "xml", "md", "cluster_size", "o", "ts", "openlid_v2", "de", "c", "text", "crawl_id", "s", "rs", "pii", "filter", "glotlid_v3", "id", "htmllang", "f"], "optional": []}, "name": "taq_Tfng", "t/d": 7784.4, "c/t": 0.9426031550280047, "samples": "https://data.hplt-project.org/three/sorted/taq_Tfng.json", "urls": ["https://data.hplt-project.org/three/sorted/taq_Tfng/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/taq_Tfng/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/taq_Tfng/8_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/taq_Tfng.map", "md5": "https://data.hplt-project.org/three/sorted/taq_Tfng.md5"}
{"files": 6, "bytes": 2343137024, "documents": 1297901, "segments": 23344597, "tokens": 1571386078, "characters": 3701788939, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["md", "cluster_size", "xml", "glotlid_v3", "pii", "text", "u", "doc_scores", "id", "de", "openlid_v3", "c", "f", "o", "ts", "s", "openlid_v2", "seg_langs_openlid_v3", "crawl_id", "rs", "filter"], "optional": ["htmllang", "metalang", "allowed"]}, "name": "tat_Cyrl", "t/d": 1210.7133579525712, "c/t": 2.355747572685317, "samples": "https://data.hplt-project.org/three/sorted/tat_Cyrl.json", "urls": ["https://data.hplt-project.org/three/sorted/tat_Cyrl/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/tat_Cyrl/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/tat_Cyrl/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/tat_Cyrl/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/tat_Cyrl/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/tat_Cyrl/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/tat_Cyrl.map", "md5": "https://data.hplt-project.org/three/sorted/tat_Cyrl.md5"}
{"files": 6, "bytes": 14737430398, "documents": 6802199, "segments": 87810507, "tokens": 6321327072, "characters": 17148987471, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["seg_langs_openlid_v3", "de", "rs", "s", "cluster_size", "ts", "md", "doc_scores", "crawl_id", "text", "glotlid_v3", "filter", "id", "u", "c", "openlid_v3", "pii", "openlid_v2", "xml", "o", "web-register", "f"], "optional": ["htmllang", "allowed", "metalang"]}, "name": "tel_Telu", "t/d": 929.3064010623623, "c/t": 2.7128777352718507, "samples": "https://data.hplt-project.org/three/sorted/tel_Telu.json", "urls": ["https://data.hplt-project.org/three/sorted/tel_Telu/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/tel_Telu/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/tel_Telu/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/tel_Telu/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/tel_Telu/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/tel_Telu/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/tel_Telu.map", "md5": "https://data.hplt-project.org/three/sorted/tel_Telu.md5"}
{"files": 6, "bytes": 2445946839, "documents": 1101345, "segments": 20048735, "tokens": 1605873922, "characters": 3728516045, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["de", "id", "text", "seg_langs_openlid_v3", "o", "cluster_size", "rs", "md", "pii", "c", "s", "xml", "crawl_id", "filter", "f", "ts", "openlid_v3", "glotlid_v3", "doc_scores", "u", "openlid_v2"], "optional": ["htmllang", "allowed", "metalang"]}, "name": "tgk_Cyrl", "t/d": 1458.1025219163841, "c/t": 2.3217987376969185, "samples": "https://data.hplt-project.org/three/sorted/tgk_Cyrl.json", "urls": ["https://data.hplt-project.org/three/sorted/tgk_Cyrl/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/tgk_Cyrl/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/tgk_Cyrl/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/tgk_Cyrl/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/tgk_Cyrl/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/tgk_Cyrl/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/tgk_Cyrl.map", "md5": "https://data.hplt-project.org/three/sorted/tgk_Cyrl.md5"}
{"files": 18, "bytes": 129874042568, "documents": 44241201, "segments": 603963041, "tokens": 57080863728, "characters": 161532472611, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["md", "openlid_v2", "id", "filter", "crawl_id", "o", "openlid_v3", "cluster_size", "pii", "de", "rs", "f", "seg_langs_openlid_v3", "text", "glotlid_v3", "s", "c", "ts", "web-register", "doc_scores", "u", "xml"], "optional": ["metalang", "htmllang", "allowed"]}, "name": "tha_Thai", "t/d": 1290.2195789847567, "c/t": 2.8298883734613693, "samples": "https://data.hplt-project.org/three/sorted/tha_Thai.json", "urls": ["https://data.hplt-project.org/three/sorted/tha_Thai/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/tha_Thai/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/tha_Thai/5_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/tha_Thai/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/tha_Thai/6_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/tha_Thai/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/tha_Thai/7_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/tha_Thai/7_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/tha_Thai/7_4.jsonl.zst", "https://data.hplt-project.org/three/sorted/tha_Thai/7_5.jsonl.zst", "https://data.hplt-project.org/three/sorted/tha_Thai/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/tha_Thai/8_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/tha_Thai/8_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/tha_Thai/8_4.jsonl.zst", "https://data.hplt-project.org/three/sorted/tha_Thai/8_5.jsonl.zst", "https://data.hplt-project.org/three/sorted/tha_Thai/8_6.jsonl.zst", "https://data.hplt-project.org/three/sorted/tha_Thai/9_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/tha_Thai/9_2.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/tha_Thai.map", "md5": "https://data.hplt-project.org/three/sorted/tha_Thai.md5"}
{"files": 6, "bytes": 191174289, "documents": 71778, "segments": 1295831, "tokens": 146593321, "characters": 203227932, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["rs", "s", "id", "u", "doc_scores", "openlid_v2", "text", "f", "glotlid_v3", "ts", "de", "c", "seg_langs_openlid_v3", "md", "xml", "cluster_size", "o", "filter", "openlid_v3", "pii", "crawl_id"], "optional": ["htmllang", "allowed", "metalang"]}, "name": "tir_Ethi", "t/d": 2042.3154866393602, "c/t": 1.3863382766258499, "samples": "https://data.hplt-project.org/three/sorted/tir_Ethi.json", "urls": ["https://data.hplt-project.org/three/sorted/tir_Ethi/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/tir_Ethi/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/tir_Ethi/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/tir_Ethi/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/tir_Ethi/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/tir_Ethi/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/tir_Ethi.map", "md5": "https://data.hplt-project.org/three/sorted/tir_Ethi.md5"}
{"files": 6, "bytes": 12460432, "documents": 5661, "segments": 124751, "tokens": 10609483, "characters": 32097617, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["seg_langs_openlid_v3", "cluster_size", "id", "rs", "s", "pii", "de", "md", "crawl_id", "openlid_v2", "o", "c", "text", "f", "ts", "doc_scores", "u", "glotlid_v3", "filter", "xml", "openlid_v3"], "optional": ["htmllang", "allowed", "metalang"]}, "name": "tpi_Latn", "t/d": 1874.135841724077, "c/t": 3.0253705105140374, "samples": "https://data.hplt-project.org/three/sorted/tpi_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/tpi_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/tpi_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/tpi_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/tpi_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/tpi_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/tpi_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/tpi_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/tpi_Latn.md5"}
{"files": 6, "bytes": 27466603, "documents": 10477, "segments": 259068, "tokens": 21927036, "characters": 60846595, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["text", "id", "md", "rs", "s", "doc_scores", "xml", "u", "c", "openlid_v2", "seg_langs_openlid_v3", "cluster_size", "crawl_id", "glotlid_v3", "filter", "f", "o", "openlid_v3", "ts", "de", "pii"], "optional": ["metalang", "htmllang", "allowed"]}, "name": "tsn_Latn", "t/d": 2092.8735324997615, "c/t": 2.7749575911673605, "samples": "https://data.hplt-project.org/three/sorted/tsn_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/tsn_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/tsn_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/tsn_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/tsn_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/tsn_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/tsn_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/tsn_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/tsn_Latn.md5"}
{"files": 6, "bytes": 20097299, "documents": 8707, "segments": 229050, "tokens": 16427240, "characters": 43511617, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["rs", "c", "u", "filter", "f", "de", "text", "openlid_v2", "pii", "doc_scores", "crawl_id", "cluster_size", "xml", "ts", "o", "id", "md", "seg_langs_openlid_v3", "glotlid_v3", "s", "openlid_v3"], "optional": ["metalang", "htmllang", "allowed"]}, "name": "tso_Latn", "t/d": 1886.6704950040198, "c/t": 2.6487478724362705, "samples": "https://data.hplt-project.org/three/sorted/tso_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/tso_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/tso_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/tso_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/tso_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/tso_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/tso_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/tso_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/tso_Latn.md5"}
{"files": 6, "bytes": 679246652, "documents": 458344, "segments": 6643823, "tokens": 522482888, "characters": 1285760561, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["u", "c", "cluster_size", "crawl_id", "id", "text", "o", "seg_langs_openlid_v3", "rs", "xml", "openlid_v2", "md", "openlid_v3", "f", "filter", "doc_scores", "de", "pii", "s", "ts", "glotlid_v3"], "optional": ["htmllang", "metalang", "allowed"]}, "name": "tuk_Latn", "t/d": 1139.9361353044874, "c/t": 2.46086635664133, "samples": "https://data.hplt-project.org/three/sorted/tuk_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/tuk_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/tuk_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/tuk_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/tuk_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/tuk_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/tuk_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/tuk_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/tuk_Latn.md5"}
{"files": 6, "bytes": 10412894, "documents": 3700, "segments": 119808, "tokens": 8475835, "characters": 22127239, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["text", "u", "ts", "cluster_size", "c", "de", "filter", "pii", "md", "xml", "rs", "id", "o", "openlid_v2", "openlid_v3", "doc_scores", "glotlid_v3", "f", "seg_langs_openlid_v3", "s", "crawl_id"], "optional": ["allowed", "metalang", "htmllang"]}, "name": "tum_Latn", "t/d": 2290.7662162162164, "c/t": 2.610626445654027, "samples": "https://data.hplt-project.org/three/sorted/tum_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/tum_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/tum_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/tum_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/tum_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/tum_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/tum_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/tum_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/tum_Latn.md5"}
{"files": 26, "bytes": 349665222444, "documents": 172888172, "segments": 3253791171, "tokens": 157553817336, "characters": 543223563754, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["text", "xml", "md", "f", "o", "s", "rs", "u", "c", "ts", "de", "id", "openlid_v2", "glotlid_v3", "openlid_v3", "seg_langs_openlid_v3", "cluster_size", "web-register", "crawl_id", "filter", "pii", "doc_scores", "bsc-edu", "finepdfs-edu", "fineweb2-hq", "jql", "propella-4b"], "optional": ["htmllang", "metalang", "allowed"]}, "name": "tur_Latn", "t/d": 911.3047787676302, "c/t": 3.4478603751981387, "samples": "https://data.hplt-project.org/three/sorted/tur_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/tur_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/tur_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/tur_Latn/5_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/tur_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/tur_Latn/6_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/tur_Latn/6_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/tur_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/tur_Latn/7_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/tur_Latn/7_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/tur_Latn/7_4.jsonl.zst", "https://data.hplt-project.org/three/sorted/tur_Latn/7_5.jsonl.zst", "https://data.hplt-project.org/three/sorted/tur_Latn/7_6.jsonl.zst", "https://data.hplt-project.org/three/sorted/tur_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/tur_Latn/8_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/tur_Latn/8_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/tur_Latn/8_4.jsonl.zst", "https://data.hplt-project.org/three/sorted/tur_Latn/8_5.jsonl.zst", "https://data.hplt-project.org/three/sorted/tur_Latn/8_6.jsonl.zst", "https://data.hplt-project.org/three/sorted/tur_Latn/8_7.jsonl.zst", "https://data.hplt-project.org/three/sorted/tur_Latn/8_8.jsonl.zst", "https://data.hplt-project.org/three/sorted/tur_Latn/9_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/tur_Latn/9_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/tur_Latn/9_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/tur_Latn/9_4.jsonl.zst", "https://data.hplt-project.org/three/sorted/tur_Latn/9_5.jsonl.zst", "https://data.hplt-project.org/three/sorted/tur_Latn/9_6.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/tur_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/tur_Latn.md5"}
{"files": 6, "bytes": 20361753, "documents": 8638, "segments": 236744, "tokens": 16329332, "characters": 37242777, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["filter", "cluster_size", "glotlid_v3", "f", "id", "seg_langs_openlid_v3", "s", "text", "md", "c", "o", "pii", "de", "ts", "xml", "doc_scores", "u", "rs", "openlid_v3", "openlid_v2", "crawl_id"], "optional": ["htmllang", "allowed", "metalang"]}, "name": "twi_Latn", "t/d": 1890.4065755962029, "c/t": 2.2807287524070183, "samples": "https://data.hplt-project.org/three/sorted/twi_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/twi_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/twi_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/twi_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/twi_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/twi_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/twi_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/twi_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/twi_Latn.md5"}
{"files": 6, "bytes": 786198072, "documents": 366617, "segments": 5123542, "tokens": 672305729, "characters": 1206119918, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["u", "c", "openlid_v2", "web-register", "glotlid_v3", "xml", "rs", "md", "pii", "ts", "text", "openlid_v3", "seg_langs_openlid_v3", "id", "filter", "doc_scores", "f", "o", "de", "s", "crawl_id", "cluster_size"], "optional": ["allowed", "htmllang", "metalang"]}, "name": "uig_Arab", "t/d": 1833.8094769200554, "c/t": 1.7940051169785585, "samples": "https://data.hplt-project.org/three/sorted/uig_Arab.json", "urls": ["https://data.hplt-project.org/three/sorted/uig_Arab/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/uig_Arab/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/uig_Arab/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/uig_Arab/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/uig_Arab/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/uig_Arab/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/uig_Arab.map", "md5": "https://data.hplt-project.org/three/sorted/uig_Arab.md5"}
{"files": 21, "bytes": 222631155145, "documents": 92929623, "segments": 1815937723, "tokens": 91383811726, "characters": 275154000592, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["text", "xml", "md", "f", "o", "s", "rs", "u", "c", "ts", "de", "id", "openlid_v2", "glotlid_v3", "openlid_v3", "seg_langs_openlid_v3", "cluster_size", "web-register", "crawl_id", "filter", "pii", "doc_scores", "bsc-edu", "finepdfs-edu", "jql", "propella-4b"], "optional": ["htmllang", "metalang", "allowed"]}, "name": "ukr_Cyrl", "t/d": 983.3657855902418, "c/t": 3.0109709301359198, "samples": "https://data.hplt-project.org/three/sorted/ukr_Cyrl.json", "urls": ["https://data.hplt-project.org/three/sorted/ukr_Cyrl/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ukr_Cyrl/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ukr_Cyrl/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ukr_Cyrl/6_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/ukr_Cyrl/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ukr_Cyrl/7_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/ukr_Cyrl/7_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/ukr_Cyrl/7_4.jsonl.zst", "https://data.hplt-project.org/three/sorted/ukr_Cyrl/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ukr_Cyrl/8_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/ukr_Cyrl/8_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/ukr_Cyrl/8_4.jsonl.zst", "https://data.hplt-project.org/three/sorted/ukr_Cyrl/8_5.jsonl.zst", "https://data.hplt-project.org/three/sorted/ukr_Cyrl/8_6.jsonl.zst", "https://data.hplt-project.org/three/sorted/ukr_Cyrl/8_7.jsonl.zst", "https://data.hplt-project.org/three/sorted/ukr_Cyrl/9_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ukr_Cyrl/9_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/ukr_Cyrl/9_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/ukr_Cyrl/9_4.jsonl.zst", "https://data.hplt-project.org/three/sorted/ukr_Cyrl/9_5.jsonl.zst", "https://data.hplt-project.org/three/sorted/ukr_Cyrl/9_6.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/ukr_Cyrl.map", "md5": "https://data.hplt-project.org/three/sorted/ukr_Cyrl.md5"}
{"files": 5, "bytes": 1354782, "documents": 623, "segments": 18795, "tokens": 1020042, "characters": 2651492, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["xml", "o", "id", "md", "seg_langs_openlid_v3", "f", "ts", "filter", "pii", "u", "c", "crawl_id", "openlid_v2", "doc_scores", "cluster_size", "de", "rs", "s", "text", "openlid_v3", "glotlid_v3"], "optional": ["metalang", "htmllang", "allowed"]}, "name": "umb_Latn", "t/d": 1637.30658105939, "c/t": 2.59939492687556, "samples": "https://data.hplt-project.org/three/sorted/umb_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/umb_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/umb_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/umb_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/umb_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/umb_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/umb_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/umb_Latn.md5"}
{"files": 6, "bytes": 14359682858, "documents": 7683866, "segments": 100945768, "tokens": 6467975035, "characters": 20375388997, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["f", "s", "de", "doc_scores", "glotlid_v3", "md", "ts", "rs", "o", "xml", "openlid_v2", "pii", "openlid_v3", "web-register", "u", "seg_langs_openlid_v3", "id", "c", "text", "crawl_id", "filter", "cluster_size"], "optional": ["htmllang", "metalang", "allowed"]}, "name": "urd_Arab", "t/d": 841.7605193791771, "c/t": 3.150195986648548, "samples": "https://data.hplt-project.org/three/sorted/urd_Arab.json", "urls": ["https://data.hplt-project.org/three/sorted/urd_Arab/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/urd_Arab/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/urd_Arab/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/urd_Arab/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/urd_Arab/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/urd_Arab/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/urd_Arab.map", "md5": "https://data.hplt-project.org/three/sorted/urd_Arab.md5"}
{"files": 6, "bytes": 4352435146, "documents": 2223989, "segments": 40622706, "tokens": 2867072229, "characters": 8131576521, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["glotlid_v3", "filter", "u", "doc_scores", "s", "md", "rs", "seg_langs_openlid_v3", "openlid_v2", "c", "crawl_id", "text", "pii", "id", "web-register", "xml", "o", "ts", "de", "openlid_v3", "f", "cluster_size"], "optional": ["htmllang", "metalang", "allowed"]}, "name": "uzn_Latn", "t/d": 1289.1575583332472, "c/t": 2.836195209437118, "samples": "https://data.hplt-project.org/three/sorted/uzn_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/uzn_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/uzn_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/uzn_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/uzn_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/uzn_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/uzn_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/uzn_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/uzn_Latn.md5"}
{"files": 6, "bytes": 58297337, "documents": 32540, "segments": 656322, "tokens": 29846169, "characters": 90918881, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["id", "openlid_v3", "crawl_id", "f", "openlid_v2", "glotlid_v3", "text", "seg_langs_openlid_v3", "de", "pii", "u", "filter", "xml", "md", "doc_scores", "c", "o", "rs", "ts", "cluster_size", "s"], "optional": ["metalang", "htmllang", "allowed"]}, "name": "vec_Latn", "t/d": 917.2147818070067, "c/t": 3.0462496208474863, "samples": "https://data.hplt-project.org/three/sorted/vec_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/vec_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/vec_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/vec_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/vec_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/vec_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/vec_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/vec_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/vec_Latn.md5"}
{"files": 25, "bytes": 325869400154, "documents": 152529479, "segments": 3754229255, "tokens": 149377735613, "characters": 500414404131, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["u", "pii", "xml", "rs", "ts", "crawl_id", "openlid_v2", "filter", "c", "web-register", "o", "seg_langs_openlid_v3", "de", "text", "f", "md", "id", "openlid_v3", "cluster_size", "s", "glotlid_v3", "doc_scores"], "optional": ["allowed", "htmllang", "metalang"]}, "name": "vie_Latn", "t/d": 979.3368245426184, "c/t": 3.349993237462425, "samples": "https://data.hplt-project.org/three/sorted/vie_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/vie_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/vie_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/vie_Latn/5_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/vie_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/vie_Latn/6_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/vie_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/vie_Latn/7_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/vie_Latn/7_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/vie_Latn/7_4.jsonl.zst", "https://data.hplt-project.org/three/sorted/vie_Latn/7_5.jsonl.zst", "https://data.hplt-project.org/three/sorted/vie_Latn/7_6.jsonl.zst", "https://data.hplt-project.org/three/sorted/vie_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/vie_Latn/8_10.jsonl.zst", "https://data.hplt-project.org/three/sorted/vie_Latn/8_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/vie_Latn/8_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/vie_Latn/8_4.jsonl.zst", "https://data.hplt-project.org/three/sorted/vie_Latn/8_5.jsonl.zst", "https://data.hplt-project.org/three/sorted/vie_Latn/8_6.jsonl.zst", "https://data.hplt-project.org/three/sorted/vie_Latn/8_7.jsonl.zst", "https://data.hplt-project.org/three/sorted/vie_Latn/8_8.jsonl.zst", "https://data.hplt-project.org/three/sorted/vie_Latn/8_9.jsonl.zst", "https://data.hplt-project.org/three/sorted/vie_Latn/9_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/vie_Latn/9_2.jsonl.zst", "https://data.hplt-project.org/three/sorted/vie_Latn/9_3.jsonl.zst", "https://data.hplt-project.org/three/sorted/vie_Latn/9_4.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/vie_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/vie_Latn.md5"}
{"files": 6, "bytes": 11655843, "documents": 9076, "segments": 110276, "tokens": 7227013, "characters": 22981017, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["pii", "cluster_size", "openlid_v3", "xml", "seg_langs_openlid_v3", "crawl_id", "glotlid_v3", "doc_scores", "rs", "id", "ts", "text", "c", "s", "de", "f", "openlid_v2", "md", "u", "o", "filter"], "optional": ["htmllang", "metalang", "allowed"]}, "name": "war_Latn", "t/d": 796.2773248126928, "c/t": 3.1798776340930894, "samples": "https://data.hplt-project.org/three/sorted/war_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/war_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/war_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/war_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/war_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/war_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/war_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/war_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/war_Latn.md5"}
{"files": 5, "bytes": 10944127, "documents": 5039, "segments": 153144, "tokens": 7656517, "characters": 18779967, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["de", "c", "md", "xml", "seg_langs_openlid_v3", "f", "o", "crawl_id", "rs", "s", "u", "glotlid_v3", "cluster_size", "doc_scores", "openlid_v2", "ts", "openlid_v3", "text", "id", "pii", "filter"], "optional": ["metalang", "allowed", "htmllang"]}, "name": "wol_Latn", "t/d": 1519.4516769200238, "c/t": 2.452808111051017, "samples": "https://data.hplt-project.org/three/sorted/wol_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/wol_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/wol_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/wol_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/wol_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/wol_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/wol_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/wol_Latn.md5"}
{"files": 6, "bytes": 577847347, "documents": 301236, "segments": 7924012, "tokens": 392306047, "characters": 1028590506, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["filter", "doc_scores", "o", "id", "u", "text", "md", "c", "glotlid_v3", "s", "web-register", "f", "openlid_v3", "xml", "openlid_v2", "rs", "crawl_id", "pii", "de", "seg_langs_openlid_v3", "ts", "cluster_size"], "optional": ["allowed", "htmllang", "metalang"]}, "name": "xho_Latn", "t/d": 1302.3212597431914, "c/t": 2.621908364313334, "samples": "https://data.hplt-project.org/three/sorted/xho_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/xho_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/xho_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/xho_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/xho_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/xho_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/xho_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/xho_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/xho_Latn.md5"}
{"files": 6, "bytes": 745378684, "documents": 284415, "segments": 6416707, "tokens": 551738901, "characters": 1083232559, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["text", "web-register", "u", "ts", "doc_scores", "glotlid_v3", "openlid_v3", "s", "filter", "id", "o", "c", "cluster_size", "seg_langs_openlid_v3", "xml", "md", "openlid_v2", "pii", "de", "rs", "f", "crawl_id"], "optional": ["allowed", "metalang", "htmllang"]}, "name": "ydd_Hebr", "t/d": 1939.9078846052423, "c/t": 1.9633064789100307, "samples": "https://data.hplt-project.org/three/sorted/ydd_Hebr.json", "urls": ["https://data.hplt-project.org/three/sorted/ydd_Hebr/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ydd_Hebr/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ydd_Hebr/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ydd_Hebr/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ydd_Hebr/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/ydd_Hebr/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/ydd_Hebr.map", "md5": "https://data.hplt-project.org/three/sorted/ydd_Hebr.md5"}
{"files": 6, "bytes": 468000746, "documents": 202101, "segments": 5116221, "tokens": 355551892, "characters": 872371816, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["filter", "u", "c", "id", "md", "doc_scores", "s", "xml", "o", "seg_langs_openlid_v3", "ts", "openlid_v2", "rs", "cluster_size", "text", "openlid_v3", "crawl_id", "f", "glotlid_v3", "de", "pii"], "optional": ["allowed", "htmllang", "metalang"]}, "name": "yor_Latn", "t/d": 1759.2782420670853, "c/t": 2.45357101348233, "samples": "https://data.hplt-project.org/three/sorted/yor_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/yor_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/yor_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/yor_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/yor_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/yor_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/yor_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/yor_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/yor_Latn.md5"}
{"files": 5, "bytes": 642258388, "documents": 281030, "segments": 5819407, "tokens": 272700131, "characters": 348764630, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["md", "openlid_v3", "de", "c", "pii", "text", "f", "o", "filter", "cluster_size", "xml", "crawl_id", "u", "id", "seg_langs_openlid_v3", "ts", "s", "web-register", "rs", "doc_scores", "glotlid_v3", "openlid_v2"], "optional": ["metalang", "allowed", "htmllang"]}, "name": "yue_Hant", "t/d": 970.3595025442123, "c/t": 1.2789309221123917, "samples": "https://data.hplt-project.org/three/sorted/yue_Hant.json", "urls": ["https://data.hplt-project.org/three/sorted/yue_Hant/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/yue_Hant/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/yue_Hant/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/yue_Hant/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/yue_Hant/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/yue_Hant.map", "md5": "https://data.hplt-project.org/three/sorted/yue_Hant.md5"}
{"files": 5, "bytes": 708520, "documents": 474, "segments": 3989, "tokens": 856580, "characters": 841482, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["de", "cluster_size", "ts", "f", "c", "md", "rs", "crawl_id", "id", "glotlid_v3", "xml", "filter", "u", "o", "openlid_v3", "s", "text", "seg_langs_openlid_v3", "doc_scores", "openlid_v2", "pii"], "optional": ["metalang", "allowed", "htmllang"]}, "name": "zgh_Tfng", "t/d": 1807.1308016877638, "c/t": 0.9823740923206239, "samples": "https://data.hplt-project.org/three/sorted/zgh_Tfng.json", "urls": ["https://data.hplt-project.org/three/sorted/zgh_Tfng/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/zgh_Tfng/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/zgh_Tfng/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/zgh_Tfng/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/zgh_Tfng/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/zgh_Tfng.map", "md5": "https://data.hplt-project.org/three/sorted/zgh_Tfng.md5"}
{"files": 6, "bytes": 36531117641, "documents": 16407730, "segments": 452972316, "tokens": 16823083359, "characters": 66083413011, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["c", "xml", "openlid_v3", "u", "f", "pii", "o", "id", "web-register", "text", "de", "ts", "doc_scores", "filter", "openlid_v2", "md", "cluster_size", "seg_langs_openlid_v3", "crawl_id", "rs", "s", "glotlid_v3"], "optional": ["allowed", "htmllang", "metalang"]}, "name": "zsm_Latn", "t/d": 1025.3144925592999, "c/t": 3.9281391883282053, "samples": "https://data.hplt-project.org/three/sorted/zsm_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/zsm_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/zsm_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/zsm_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/zsm_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/zsm_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/zsm_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/zsm_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/zsm_Latn.md5"}
{"files": 6, "bytes": 651434966, "documents": 367119, "segments": 9037930, "tokens": 456139670, "characters": 1237579038, "tokenizer": "google/gemma-3-4b-it", "keys": {"required": ["c", "cluster_size", "id", "seg_langs_openlid_v3", "text", "xml", "glotlid_v3", "ts", "pii", "f", "doc_scores", "rs", "crawl_id", "o", "s", "md", "openlid_v3", "u", "openlid_v2", "filter", "de"], "optional": ["htmllang", "allowed", "metalang"]}, "name": "zul_Latn", "t/d": 1242.4845077481689, "c/t": 2.7131580947563716, "samples": "https://data.hplt-project.org/three/sorted/zul_Latn.json", "urls": ["https://data.hplt-project.org/three/sorted/zul_Latn/10_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/zul_Latn/5_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/zul_Latn/6_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/zul_Latn/7_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/zul_Latn/8_1.jsonl.zst", "https://data.hplt-project.org/three/sorted/zul_Latn/9_1.jsonl.zst"], "map": "https://data.hplt-project.org/three/sorted/zul_Latn.map", "md5": "https://data.hplt-project.org/three/sorted/zul_Latn.md5"}