import{s as vn,o as yn,n as C}from"../chunks/scheduler.7da89386.js";import{S as bn,i as Mn,g as p,s as l,r as g,A as wn,h as u,f as i,c as r,j as _,u as h,x as $,k as j,y as n,a as M,v as f,d as v,t as y,w as b}from"../chunks/index.20910acc.js";import{T as qa}from"../chunks/Tip.53e22153.js";import{D as x}from"../chunks/Docstring.f0851586.js";import{C as I}from"../chunks/CodeBlock.143bd81e.js";import{E as k}from"../chunks/ExampleCodeBlock.d74eb35c.js";import{H as W,E as $n}from"../chunks/getInferenceSnippets.217b4024.js";function Tn(T){let a,m="Examples:",s,o,d;return o=new I({props:{code:"ZnJvbSUyMGV2YWx1YXRlJTIwaW1wb3J0JTIwZXZhbHVhdG9yJTBBJTIzJTIwU2VudGltZW50JTIwYW5hbHlzaXMlMjBldmFsdWF0b3IlMEFldmFsdWF0b3IoJTIyc2VudGltZW50LWFuYWx5c2lzJTIyKQ==",highlighted:`>>> from evaluate import evaluator
>>>
>>> evaluator("sentiment-analysis")`,wrap:!1}}),{c(){a=p("p"),a.textContent=m,s=l(),g(o.$$.fragment)},l(t){a=u(t,"P",{"data-svelte-h":!0}),$(a)!=="svelte-kvfsh7"&&(a.textContent=m),s=r(t),h(o.$$.fragment,t)},m(t,w){M(t,a,w),M(t,s,w),f(o,t,w),d=!0},p:C,i(t){d||(v(o.$$.fragment,t),d=!0)},o(t){y(o.$$.fragment,t),d=!1},d(t){t&&(i(a),i(s)),b(o,t)}}}function _n(T){let a,m="Example:",s,o,d;return o=new I({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9kYXRhc2V0JTBBZnJvbSUyMGV2YWx1YXRlJTIwaW1wb3J0JTIwZXZhbHVhdG9yJTBBZGF0YSUyMCUzRCUyMGxvYWRfZGF0YXNldCglMjJyb3R0ZW5fdG9tYXRvZXMnJTJDJTIwc3BsaXQlM0QlMjJ0cmFpbiUyMiklMEFldmFsdWF0b3IuY2hlY2tfcmVxdWlyZWRfY29sdW1ucyhkYXRhJTJDJTIwJTdCJTIyaW5wdXRfY29sdW1uJTIyJTNBJTIwJTIydGV4dCUyMiUyQyUyMCUyMmxhYmVsX2NvbHVtbiUyMiUzQSUyMCUyMmxhYmVsJTIyJTdEKQ==",highlighted:`>>> from datasets import load_dataset
>>> from evaluate import evaluator
>>> data = load_dataset("rotten_tomatoes', split="train")
>>> evaluator.check_required_columns(data, {"input_column": "text", "label_column": "label"})`,wrap:!1}}),{c(){a=p("p"),a.textContent=m,s=l(),g(o.$$.fragment)},l(t){a=u(t,"P",{"data-svelte-h":!0}),$(a)!=="svelte-11lpom8"&&(a.textContent=m),s=r(t),h(o.$$.fragment,t)},m(t,w){M(t,a,w),M(t,s,w),f(o,t,w),d=!0},p:C,i(t){d||(v(o.$$.fragment,t),d=!0)},o(t){y(o.$$.fragment,t),d=!1},d(t){t&&(i(a),i(s)),b(o,t)}}}function jn(T){let a,m="Example:",s,o,d;return o=new I({props:{code:"ZnJvbSUyMGV2YWx1YXRlJTIwaW1wb3J0JTIwZXZhbHVhdG9yJTBBZXZhbHVhdG9yKCUyMnRleHQtY2xhc3NpZmljYXRpb24lMjIpLmdldF9kYXRhc2V0X3NwbGl0KGRhdGElM0QlMjJyb3R0ZW5fdG9tYXRvZXMlMjIp",highlighted:`>>> from evaluate import evaluator
>>> evaluator("text-classification").get_dataset_split(data="rotten_tomatoes")
WARNING:evaluate.evaluator.base:Dataset split not defined! Automatically evaluating with split: TEST
'test'`,wrap:!1}}),{c(){a=p("p"),a.textContent=m,s=l(),g(o.$$.fragment)},l(t){a=u(t,"P",{"data-svelte-h":!0}),$(a)!=="svelte-11lpom8"&&(a.textContent=m),s=r(t),h(o.$$.fragment,t)},m(t,w){M(t,a,w),M(t,s,w),f(o,t,w),d=!0},p:C,i(t){d||(v(o.$$.fragment,t),d=!0)},o(t){y(o.$$.fragment,t),d=!1},d(t){t&&(i(a),i(s)),b(o,t)}}}function xn(T){let a,m="Example:",s,o,d;return o=new I({props:{code:"ZnJvbSUyMGV2YWx1YXRlJTIwaW1wb3J0JTIwZXZhbHVhdG9yJTBBZXZhbHVhdG9yKCUyMnRleHQtY2xhc3NpZmljYXRpb24lMjIpLmxvYWRfZGF0YShkYXRhJTNEJTIycm90dGVuX3RvbWF0b2VzJTIyJTJDJTIwc3BsaXQlM0QlMjJ0cmFpbiUyMik=",highlighted:`>>> from evaluate import evaluator
>>> evaluator("text-classification").load_data(data="rotten_tomatoes", split="train")
Dataset({
features: ['text', 'label'],
num_rows: 8530
})`,wrap:!1}}),{c(){a=p("p"),a.textContent=m,s=l(),g(o.$$.fragment)},l(t){a=u(t,"P",{"data-svelte-h":!0}),$(a)!=="svelte-11lpom8"&&(a.textContent=m),s=r(t),h(o.$$.fragment,t)},m(t,w){M(t,a,w),M(t,s,w),f(o,t,w),d=!0},p:C,i(t){d||(v(o.$$.fragment,t),d=!0)},o(t){y(o.$$.fragment,t),d=!1},d(t){t&&(i(a),i(s)),b(o,t)}}}function Jn(T){let a,m="Example:",s,o,d;return o=new I({props:{code:"ZnJvbSUyMGV2YWx1YXRlJTIwaW1wb3J0JTIwZXZhbHVhdG9yJTBBZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9kYXRhc2V0JTBBJTBBZHMlMjAlM0QlMjBsb2FkX2RhdGFzZXQoJTIycm90dGVuX3RvbWF0b2VzJTIyJTJDJTIwc3BsaXQlM0QlMjJ0cmFpbiUyMiklMEFldmFsdWF0b3IoJTIydGV4dC1jbGFzc2lmaWNhdGlvbiUyMikucHJlcGFyZV9kYXRhKGRzJTJDJTIwaW5wdXRfY29sdW1uJTNEJTIydGV4dCUyMiUyQyUyMHNlY29uZF9pbnB1dF9jb2x1bW4lM0ROb25lJTJDJTIwbGFiZWxfY29sdW1uJTNEJTIybGFiZWwlMjIp",highlighted:`>>> from evaluate import evaluator
>>> from datasets import load_dataset
>>> ds = load_dataset("rotten_tomatoes", split="train")
>>> evaluator("text-classification").prepare_data(ds, input_column="text", second_input_column=None, label_column="label")`,wrap:!1}}),{c(){a=p("p"),a.textContent=m,s=l(),g(o.$$.fragment)},l(t){a=u(t,"P",{"data-svelte-h":!0}),$(a)!=="svelte-11lpom8"&&(a.textContent=m),s=r(t),h(o.$$.fragment,t)},m(t,w){M(t,a,w),M(t,s,w),f(o,t,w),d=!0},p:C,i(t){d||(v(o.$$.fragment,t),d=!0)},o(t){y(o.$$.fragment,t),d=!1},d(t){t&&(i(a),i(s)),b(o,t)}}}function Un(T){let a,m="Example:",s,o,d;return o=new I({props:{code:"ZnJvbSUyMGV2YWx1YXRlJTIwaW1wb3J0JTIwZXZhbHVhdG9yJTBBZXZhbHVhdG9yKCUyMnRleHQtY2xhc3NpZmljYXRpb24lMjIpLnByZXBhcmVfbWV0cmljKCUyMmFjY3VyYWN5JTIyKQ==",highlighted:`>>> from evaluate import evaluator
>>> evaluator("text-classification").prepare_metric("accuracy")`,wrap:!1}}),{c(){a=p("p"),a.textContent=m,s=l(),g(o.$$.fragment)},l(t){a=u(t,"P",{"data-svelte-h":!0}),$(a)!=="svelte-11lpom8"&&(a.textContent=m),s=r(t),h(o.$$.fragment,t)},m(t,w){M(t,a,w),M(t,s,w),f(o,t,w),d=!0},p:C,i(t){d||(v(o.$$.fragment,t),d=!0)},o(t){y(o.$$.fragment,t),d=!1},d(t){t&&(i(a),i(s)),b(o,t)}}}function Cn(T){let a,m="Example:",s,o,d;return o=new I({props:{code:"ZnJvbSUyMGV2YWx1YXRlJTIwaW1wb3J0JTIwZXZhbHVhdG9yJTBBZXZhbHVhdG9yKCUyMnRleHQtY2xhc3NpZmljYXRpb24lMjIpLnByZXBhcmVfcGlwZWxpbmUobW9kZWxfb3JfcGlwZWxpbmUlM0QlMjJkaXN0aWxiZXJ0LWJhc2UtdW5jYXNlZCUyMik=",highlighted:`>>> from evaluate import evaluator
>>> evaluator("text-classification").prepare_pipeline(model_or_pipeline="distilbert-base-uncased")`,wrap:!1}}),{c(){a=p("p"),a.textContent=m,s=l(),g(o.$$.fragment)},l(t){a=u(t,"P",{"data-svelte-h":!0}),$(a)!=="svelte-11lpom8"&&(a.textContent=m),s=r(t),h(o.$$.fragment,t)},m(t,w){M(t,a,w),M(t,s,w),f(o,t,w),d=!0},p:C,i(t){d||(v(o.$$.fragment,t),d=!0)},o(t){y(o.$$.fragment,t),d=!1},d(t){t&&(i(a),i(s)),b(o,t)}}}function kn(T){let a,m="Examples:",s,o,d;return o=new I({props:{code:"ZnJvbSUyMGV2YWx1YXRlJTIwaW1wb3J0JTIwZXZhbHVhdG9yJTBBZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9kYXRhc2V0JTBBdGFza19ldmFsdWF0b3IlMjAlM0QlMjBldmFsdWF0b3IoJTIyaW1hZ2UtY2xhc3NpZmljYXRpb24lMjIpJTBBZGF0YSUyMCUzRCUyMGxvYWRfZGF0YXNldCglMjJiZWFucyUyMiUyQyUyMHNwbGl0JTNEJTIydGVzdCU1QiUzQTQwJTVEJTIyKSUwQXJlc3VsdHMlMjAlM0QlMjB0YXNrX2V2YWx1YXRvci5jb21wdXRlKCUwQSUyMCUyMCUyMCUyMG1vZGVsX29yX3BpcGVsaW5lJTNEJTIybmF0ZXJhdyUyRnZpdC1iYXNlLWJlYW5zJTIyJTJDJTBBJTIwJTIwJTIwJTIwZGF0YSUzRGRhdGElMkMlMEElMjAlMjAlMjAlMjBsYWJlbF9jb2x1bW4lM0QlMjJsYWJlbHMlMjIlMkMlMEElMjAlMjAlMjAlMjBtZXRyaWMlM0QlMjJhY2N1cmFjeSUyMiUyQyUwQSUyMCUyMCUyMCUyMGxhYmVsX21hcHBpbmclM0QlN0InYW5ndWxhcl9sZWFmX3Nwb3QnJTNBJTIwMCUyQyUyMCdiZWFuX3J1c3QnJTNBJTIwMSUyQyUyMCdoZWFsdGh5JyUzQSUyMDIlN0QlMkMlMEElMjAlMjAlMjAlMjBzdHJhdGVneSUzRCUyMmJvb3RzdHJhcCUyMiUwQSk=",highlighted:`>>> from evaluate import evaluator
>>> from datasets import load_dataset
>>> task_evaluator = evaluator("image-classification")
>>> data = load_dataset("beans", split="test[:40]")
>>> results = task_evaluator.compute(
>>> model_or_pipeline="nateraw/vit-base-beans",
>>> data=data,
>>> label_column="labels",
>>> metric="accuracy",
>>> label_mapping={'angular_leaf_spot': 0, 'bean_rust': 1, 'healthy': 2},
>>> strategy="bootstrap"
>>> )`,wrap:!1}}),{c(){a=p("p"),a.textContent=m,s=l(),g(o.$$.fragment)},l(t){a=u(t,"P",{"data-svelte-h":!0}),$(a)!=="svelte-kvfsh7"&&(a.textContent=m),s=r(t),h(o.$$.fragment,t)},m(t,w){M(t,a,w),M(t,s,w),f(o,t,w),d=!0},p:C,i(t){d||(v(o.$$.fragment,t),d=!0)},o(t){y(o.$$.fragment,t),d=!1},d(t){t&&(i(a),i(s)),b(o,t)}}}function In(T){let a,m="Examples:",s,o,d;return o=new I({props:{code:"ZnJvbSUyMGV2YWx1YXRlJTIwaW1wb3J0JTIwZXZhbHVhdG9yJTBBZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9kYXRhc2V0JTBBdGFza19ldmFsdWF0b3IlMjAlM0QlMjBldmFsdWF0b3IoJTIycXVlc3Rpb24tYW5zd2VyaW5nJTIyKSUwQWRhdGElMjAlM0QlMjBsb2FkX2RhdGFzZXQoJTIyc3F1YWQlMjIlMkMlMjBzcGxpdCUzRCUyMnZhbGlkYXRpb24lNUIlM0EyJTVEJTIyKSUwQXJlc3VsdHMlMjAlM0QlMjB0YXNrX2V2YWx1YXRvci5jb21wdXRlKCUwQSUyMCUyMCUyMCUyMG1vZGVsX29yX3BpcGVsaW5lJTNEJTIyc3NobGVpZmVyJTJGdGlueS1kaXN0aWxiZXJ0LWJhc2UtY2FzZWQtZGlzdGlsbGVkLXNxdWFkJTIyJTJDJTBBJTIwJTIwJTIwJTIwZGF0YSUzRGRhdGElMkMlMEElMjAlMjAlMjAlMjBtZXRyaWMlM0QlMjJzcXVhZCUyMiUyQyUwQSk=",highlighted:`>>> from evaluate import evaluator
>>> from datasets import load_dataset
>>> task_evaluator = evaluator("question-answering")
>>> data = load_dataset("squad", split="validation[:2]")
>>> results = task_evaluator.compute(
>>> model_or_pipeline="sshleifer/tiny-distilbert-base-cased-distilled-squad",
>>> data=data,
>>> metric="squad",
>>> )`,wrap:!1}}),{c(){a=p("p"),a.textContent=m,s=l(),g(o.$$.fragment)},l(t){a=u(t,"P",{"data-svelte-h":!0}),$(a)!=="svelte-kvfsh7"&&(a.textContent=m),s=r(t),h(o.$$.fragment,t)},m(t,w){M(t,a,w),M(t,s,w),f(o,t,w),d=!0},p:C,i(t){d||(v(o.$$.fragment,t),d=!0)},o(t){y(o.$$.fragment,t),d=!1},d(t){t&&(i(a),i(s)),b(o,t)}}}function En(T){let a,m=`Datasets where the answer may be missing in the context are supported, for example SQuAD v2 dataset. In this case, it is safer to pass squad_v2_format=True to
the compute() call.`;return{c(){a=p("p"),a.innerHTML=m},l(s){a=u(s,"P",{"data-svelte-h":!0}),$(a)!=="svelte-r7zuhd"&&(a.innerHTML=m)},m(s,o){M(s,a,o)},p:C,d(s){s&&i(a)}}}function qn(T){let a,m;return a=new I({props:{code:"ZnJvbSUyMGV2YWx1YXRlJTIwaW1wb3J0JTIwZXZhbHVhdG9yJTBBZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9kYXRhc2V0JTBBdGFza19ldmFsdWF0b3IlMjAlM0QlMjBldmFsdWF0b3IoJTIycXVlc3Rpb24tYW5zd2VyaW5nJTIyKSUwQWRhdGElMjAlM0QlMjBsb2FkX2RhdGFzZXQoJTIyc3F1YWRfdjIlMjIlMkMlMjBzcGxpdCUzRCUyMnZhbGlkYXRpb24lNUIlM0EyJTVEJTIyKSUwQXJlc3VsdHMlMjAlM0QlMjB0YXNrX2V2YWx1YXRvci5jb21wdXRlKCUwQSUyMCUyMCUyMCUyMG1vZGVsX29yX3BpcGVsaW5lJTNEJTIybXJtODQ4OCUyRmJlcnQtdGlueS1maW5ldHVuZWQtc3F1YWR2MiUyMiUyQyUwQSUyMCUyMCUyMCUyMGRhdGElM0RkYXRhJTJDJTBBJTIwJTIwJTIwJTIwbWV0cmljJTNEJTIyc3F1YWRfdjIlMjIlMkMlMEElMjAlMjAlMjAlMjBzcXVhZF92Ml9mb3JtYXQlM0RUcnVlJTJDJTBBKQ==",highlighted:`>>> from evaluate import evaluator
>>> from datasets import load_dataset
>>> task_evaluator = evaluator("question-answering")
>>> data = load_dataset("squad_v2", split="validation[:2]")
>>> results = task_evaluator.compute(
>>> model_or_pipeline="mrm8488/bert-tiny-finetuned-squadv2",
>>> data=data,
>>> metric="squad_v2",
>>> squad_v2_format=True,
>>> )`,wrap:!1}}),{c(){g(a.$$.fragment)},l(s){h(a.$$.fragment,s)},m(s,o){f(a,s,o),m=!0},p:C,i(s){m||(v(a.$$.fragment,s),m=!0)},o(s){y(a.$$.fragment,s),m=!1},d(s){b(a,s)}}}function Nn(T){let a,m="Examples:",s,o,d;return o=new I({props:{code:"ZnJvbSUyMGV2YWx1YXRlJTIwaW1wb3J0JTIwZXZhbHVhdG9yJTBBZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9kYXRhc2V0JTBBdGFza19ldmFsdWF0b3IlMjAlM0QlMjBldmFsdWF0b3IoJTIydGV4dC1jbGFzc2lmaWNhdGlvbiUyMiklMEFkYXRhJTIwJTNEJTIwbG9hZF9kYXRhc2V0KCUyMmltZGIlMjIlMkMlMjBzcGxpdCUzRCUyMnRlc3QlNUIlM0EyJTVEJTIyKSUwQXJlc3VsdHMlMjAlM0QlMjB0YXNrX2V2YWx1YXRvci5jb21wdXRlKCUwQSUyMCUyMCUyMCUyMG1vZGVsX29yX3BpcGVsaW5lJTNEJTIyaHVnZ2luZ2ZhY2UlMkZwcnVuZWJlcnQtYmFzZS11bmNhc2VkLTYtZmluZXBydW5lZC13LWRpc3RpbC1tbmxpJTIyJTJDJTBBJTIwJTIwJTIwJTIwZGF0YSUzRGRhdGElMkMlMEElMjAlMjAlMjAlMjBtZXRyaWMlM0QlMjJhY2N1cmFjeSUyMiUyQyUwQSUyMCUyMCUyMCUyMGxhYmVsX21hcHBpbmclM0QlN0IlMjJMQUJFTF8wJTIyJTNBJTIwMC4wJTJDJTIwJTIyTEFCRUxfMSUyMiUzQSUyMDEuMCU3RCUyQyUwQSUyMCUyMCUyMCUyMHN0cmF0ZWd5JTNEJTIyYm9vdHN0cmFwJTIyJTJDJTBBJTIwJTIwJTIwJTIwbl9yZXNhbXBsZXMlM0QxMCUyQyUwQSUyMCUyMCUyMCUyMHJhbmRvbV9zdGF0ZSUzRDAlMEEp",highlighted:`>>> from evaluate import evaluator
>>> from datasets import load_dataset
>>> task_evaluator = evaluator("text-classification")
>>> data = load_dataset("imdb", split="test[:2]")
>>> results = task_evaluator.compute(
>>> model_or_pipeline="huggingface/prunebert-base-uncased-6-finepruned-w-distil-mnli",
>>> data=data,
>>> metric="accuracy",
>>> label_mapping={"LABEL_0": 0.0, "LABEL_1": 1.0},
>>> strategy="bootstrap",
>>> n_resamples=10,
>>> random_state=0
>>> )`,wrap:!1}}),{c(){a=p("p"),a.textContent=m,s=l(),g(o.$$.fragment)},l(t){a=u(t,"P",{"data-svelte-h":!0}),$(a)!=="svelte-kvfsh7"&&(a.textContent=m),s=r(t),h(o.$$.fragment,t)},m(t,w){M(t,a,w),M(t,s,w),f(o,t,w),d=!0},p:C,i(t){d||(v(o.$$.fragment,t),d=!0)},o(t){y(o.$$.fragment,t),d=!1},d(t){t&&(i(a),i(s)),b(o,t)}}}function Gn(T){let a,m="Examples:",s,o,d;return o=new I({props:{code:"ZnJvbSUyMGV2YWx1YXRlJTIwaW1wb3J0JTIwZXZhbHVhdG9yJTBBZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9kYXRhc2V0JTBBdGFza19ldmFsdWF0b3IlMjAlM0QlMjBldmFsdWF0b3IoJTIydG9rZW4tY2xhc3NpZmljYXRpb24lMjIpJTBBZGF0YSUyMCUzRCUyMGxvYWRfZGF0YXNldCglMjJjb25sbDIwMDMlMjIlMkMlMjBzcGxpdCUzRCUyMnZhbGlkYXRpb24lNUIlM0EyJTVEJTIyKSUwQXJlc3VsdHMlMjAlM0QlMjB0YXNrX2V2YWx1YXRvci5jb21wdXRlKCUwQSUyMCUyMCUyMCUyMG1vZGVsX29yX3BpcGVsaW5lJTNEJTIyZWxhc3RpYyUyRmRpc3RpbGJlcnQtYmFzZS11bmNhc2VkLWZpbmV0dW5lZC1jb25sbDAzLWVuZ2xpc2glMjIlMkMlMEElMjAlMjAlMjAlMjBkYXRhJTNEZGF0YSUyQyUwQSUyMCUyMCUyMCUyMG1ldHJpYyUzRCUyMnNlcWV2YWwlMjIlMkMlMEEp",highlighted:`>>> from evaluate import evaluator
>>> from datasets import load_dataset
>>> task_evaluator = evaluator("token-classification")
>>> data = load_dataset("conll2003", split="validation[:2]")
>>> results = task_evaluator.compute(
>>> model_or_pipeline="elastic/distilbert-base-uncased-finetuned-conll03-english",
>>> data=data,
>>> metric="seqeval",
>>> )`,wrap:!1}}),{c(){a=p("p"),a.textContent=m,s=l(),g(o.$$.fragment)},l(t){a=u(t,"P",{"data-svelte-h":!0}),$(a)!=="svelte-kvfsh7"&&(a.textContent=m),s=r(t),h(o.$$.fragment,t)},m(t,w){M(t,a,w),M(t,s,w),f(o,t,w),d=!0},p:C,i(t){d||(v(o.$$.fragment,t),d=!0)},o(t){y(o.$$.fragment,t),d=!1},d(t){t&&(i(a),i(s)),b(o,t)}}}function Rn(T){let a,m="For example, the following dataset format is accepted by the evaluator:",s,o,d;return o=new I({props:{code:"ZGF0YXNldCUyMCUzRCUyMERhdGFzZXQuZnJvbV9kaWN0KCUwQSUyMCUyMCUyMCUyMG1hcHBpbmclM0QlN0IlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJ0b2tlbnMlMjIlM0ElMjAlNUIlNUIlMjJOZXclMjIlMkMlMjAlMjJZb3JrJTIyJTJDJTIwJTIyaXMlMjIlMkMlMjAlMjJhJTIyJTJDJTIwJTIyY2l0eSUyMiUyQyUyMCUyMmFuZCUyMiUyQyUyMCUyMkZlbGl4JTIyJTJDJTIwJTIyYSUyMiUyQyUyMCUyMnBlcnNvbiUyMiUyQyUyMCUyMi4lMjIlNUQlNUQlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJuZXJfdGFncyUyMiUzQSUyMCU1QiU1QjElMkMlMjAyJTJDJTIwMCUyQyUyMDAlMkMlMjAwJTJDJTIwMCUyQyUyMDMlMkMlMjAwJTJDJTIwMCUyQyUyMDAlNUQlNUQlMkMlMEElMjAlMjAlMjAlMjAlN0QlMkMlMEElMjAlMjAlMjAlMjBmZWF0dXJlcyUzREZlYXR1cmVzKCU3QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMnRva2VucyUyMiUzQSUyMFNlcXVlbmNlKGZlYXR1cmUlM0RWYWx1ZShkdHlwZSUzRCUyMnN0cmluZyUyMikpJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIybmVyX3RhZ3MlMjIlM0ElMjBTZXF1ZW5jZShmZWF0dXJlJTNEQ2xhc3NMYWJlbChuYW1lcyUzRCU1QiUyMk8lMjIlMkMlMjAlMjJCLUxPQyUyMiUyQyUyMCUyMkktTE9DJTIyJTJDJTIwJTIyQi1QRVIlMjIlMkMlMjAlMjJJLVBFUiUyMiU1RCkpJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTdEKSUyQyUwQSk=",highlighted:`dataset = Dataset.from_dict(
mapping={
"tokens": [["New", "York", "is", "a", "city", "and", "Felix", "a", "person", "."]],
"ner_tags": [[1, 2, 0, 0, 0, 0, 3, 0, 0, 0]],
},
features=Features({
"tokens": Sequence(feature=Value(dtype="string")),
"ner_tags": Sequence(feature=ClassLabel(names=["O", "B-LOC", "I-LOC", "B-PER", "I-PER"])),
}),
)`,wrap:!1}}),{c(){a=p("p"),a.textContent=m,s=l(),g(o.$$.fragment)},l(t){a=u(t,"P",{"data-svelte-h":!0}),$(a)!=="svelte-n4qt64"&&(a.textContent=m),s=r(t),h(o.$$.fragment,t)},m(t,w){M(t,a,w),M(t,s,w),f(o,t,w),d=!0},p:C,i(t){d||(v(o.$$.fragment,t),d=!0)},o(t){y(o.$$.fragment,t),d=!1},d(t){t&&(i(a),i(s)),b(o,t)}}}function Fn(T){let a,m;return a=new k({props:{anchor:"evaluate.TokenClassificationEvaluator.compute.example-2",$$slots:{default:[Rn]},$$scope:{ctx:T}}}),{c(){g(a.$$.fragment)},l(s){h(a.$$.fragment,s)},m(s,o){f(a,s,o),m=!0},p(s,o){const d={};o&2&&(d.$$scope={dirty:o,ctx:s}),a.$set(d)},i(s){m||(v(a.$$.fragment,s),m=!0)},o(s){y(a.$$.fragment,s),m=!1},d(s){b(a,s)}}}function zn(T){let a,m="For example, the following dataset format is not accepted by the evaluator:",s,o,d;return o=new I({props:{code:"ZGF0YXNldCUyMCUzRCUyMERhdGFzZXQuZnJvbV9kaWN0KCUwQSUyMCUyMCUyMCUyMG1hcHBpbmclM0QlN0IlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJ0b2tlbnMlMjIlM0ElMjAlNUIlNUIlMjJOZXclMjBZb3JrJTIwaXMlMjBhJTIwY2l0eSUyMGFuZCUyMEZlbGl4JTIwYSUyMHBlcnNvbi4lMjIlNUQlNUQlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJzdGFydHMlMjIlM0ElMjAlNUIlNUIwJTJDJTIwMjMlNUQlNUQlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJlbmRzJTIyJTNBJTIwJTVCJTVCNyUyQyUyMDI3JTVEJTVEJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIybmVyX3RhZ3MlMjIlM0ElMjAlNUIlNUIlMjJMT0MlMjIlMkMlMjAlMjJQRVIlMjIlNUQlNUQlMkMlMEElMjAlMjAlMjAlMjAlN0QlMkMlMEElMjAlMjAlMjAlMjBmZWF0dXJlcyUzREZlYXR1cmVzKCU3QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMnRva2VucyUyMiUzQSUyMFZhbHVlKGR0eXBlJTNEJTIyc3RyaW5nJTIyKSUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMnN0YXJ0cyUyMiUzQSUyMFNlcXVlbmNlKGZlYXR1cmUlM0RWYWx1ZShkdHlwZSUzRCUyMmludDMyJTIyKSklMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJlbmRzJTIyJTNBJTIwU2VxdWVuY2UoZmVhdHVyZSUzRFZhbHVlKGR0eXBlJTNEJTIyaW50MzIlMjIpKSUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMm5lcl90YWdzJTIyJTNBJTIwU2VxdWVuY2UoZmVhdHVyZSUzRFZhbHVlKGR0eXBlJTNEJTIyc3RyaW5nJTIyKSklMkMlMEElMjAlMjAlMjAlMjAlN0QpJTJDJTBBKQ==",highlighted:`dataset = Dataset.from_dict(
mapping={
"tokens": [["New York is a city and Felix a person."]],
"starts": [[0, 23]],
"ends": [[7, 27]],
"ner_tags": [["LOC", "PER"]],
},
features=Features({
"tokens": Value(dtype="string"),
"starts": Sequence(feature=Value(dtype="int32")),
"ends": Sequence(feature=Value(dtype="int32")),
"ner_tags": Sequence(feature=Value(dtype="string")),
}),
)`,wrap:!1}}),{c(){a=p("p"),a.innerHTML=m,s=l(),g(o.$$.fragment)},l(t){a=u(t,"P",{"data-svelte-h":!0}),$(a)!=="svelte-sh5geg"&&(a.innerHTML=m),s=r(t),h(o.$$.fragment,t)},m(t,w){M(t,a,w),M(t,s,w),f(o,t,w),d=!0},p:C,i(t){d||(v(o.$$.fragment,t),d=!0)},o(t){y(o.$$.fragment,t),d=!1},d(t){t&&(i(a),i(s)),b(o,t)}}}function Zn(T){let a,m;return a=new k({props:{anchor:"evaluate.TokenClassificationEvaluator.compute.example-3",$$slots:{default:[zn]},$$scope:{ctx:T}}}),{c(){g(a.$$.fragment)},l(s){h(a.$$.fragment,s)},m(s,o){f(a,s,o),m=!0},p(s,o){const d={};o&2&&(d.$$scope={dirty:o,ctx:s}),a.$set(d)},i(s){m||(v(a.$$.fragment,s),m=!0)},o(s){y(a.$$.fragment,s),m=!1},d(s){b(a,s)}}}function An(T){let a,m="Examples:",s,o,d;return o=new I({props:{code:"ZnJvbSUyMGV2YWx1YXRlJTIwaW1wb3J0JTIwZXZhbHVhdG9yJTBBZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9kYXRhc2V0JTBBdGFza19ldmFsdWF0b3IlMjAlM0QlMjBldmFsdWF0b3IoJTIydGV4dDJ0ZXh0LWdlbmVyYXRpb24lMjIpJTBBZGF0YSUyMCUzRCUyMGxvYWRfZGF0YXNldCglMjJjbm5fZGFpbHltYWlsJTIyJTJDJTIwJTIyMy4wLjAlMjIlMkMlMjBzcGxpdCUzRCUyMnZhbGlkYXRpb24lNUIlM0E0MCU1RCUyMiklMEFyZXN1bHRzJTIwJTNEJTIwdGFza19ldmFsdWF0b3IuY29tcHV0ZSglMEElMjAlMjAlMjAlMjBtb2RlbF9vcl9waXBlbGluZSUzRCUyMmZhY2Vib29rJTJGYmFydC1sYXJnZS1jbm4lMjIlMkMlMEElMjAlMjAlMjAlMjBkYXRhJTNEZGF0YSUyQyUwQSUyMCUyMCUyMCUyMGlucHV0X2NvbHVtbiUzRCUyMmFydGljbGUlMjIlMkMlMEElMjAlMjAlMjAlMjBsYWJlbF9jb2x1bW4lM0QlMjJoaWdobGlnaHRzJTIyJTJDJTBBJTIwJTIwJTIwJTIwbWV0cmljJTNEJTIycm91Z2UlMjIlMkMlMEEp",highlighted:`>>> from evaluate import evaluator
>>> from datasets import load_dataset
>>> task_evaluator = evaluator("text2text-generation")
>>> data = load_dataset("cnn_dailymail", "3.0.0", split="validation[:40]")
>>> results = task_evaluator.compute(
>>> model_or_pipeline="facebook/bart-large-cnn",
>>> data=data,
>>> input_column="article",
>>> label_column="highlights",
>>> metric="rouge",
>>> )`,wrap:!1}}),{c(){a=p("p"),a.textContent=m,s=l(),g(o.$$.fragment)},l(t){a=u(t,"P",{"data-svelte-h":!0}),$(a)!=="svelte-kvfsh7"&&(a.textContent=m),s=r(t),h(o.$$.fragment,t)},m(t,w){M(t,a,w),M(t,s,w),f(o,t,w),d=!0},p:C,i(t){d||(v(o.$$.fragment,t),d=!0)},o(t){y(o.$$.fragment,t),d=!1},d(t){t&&(i(a),i(s)),b(o,t)}}}function Xn(T){let a,m="Examples:",s,o,d;return o=new I({props:{code:"ZnJvbSUyMGV2YWx1YXRlJTIwaW1wb3J0JTIwZXZhbHVhdG9yJTBBZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9kYXRhc2V0JTBBdGFza19ldmFsdWF0b3IlMjAlM0QlMjBldmFsdWF0b3IoJTIyc3VtbWFyaXphdGlvbiUyMiklMEFkYXRhJTIwJTNEJTIwbG9hZF9kYXRhc2V0KCUyMmNubl9kYWlseW1haWwlMjIlMkMlMjAlMjIzLjAuMCUyMiUyQyUyMHNwbGl0JTNEJTIydmFsaWRhdGlvbiU1QiUzQTQwJTVEJTIyKSUwQXJlc3VsdHMlMjAlM0QlMjB0YXNrX2V2YWx1YXRvci5jb21wdXRlKCUwQSUyMCUyMCUyMCUyMG1vZGVsX29yX3BpcGVsaW5lJTNEJTIyZmFjZWJvb2slMkZiYXJ0LWxhcmdlLWNubiUyMiUyQyUwQSUyMCUyMCUyMCUyMGRhdGElM0RkYXRhJTJDJTBBJTIwJTIwJTIwJTIwaW5wdXRfY29sdW1uJTNEJTIyYXJ0aWNsZSUyMiUyQyUwQSUyMCUyMCUyMCUyMGxhYmVsX2NvbHVtbiUzRCUyMmhpZ2hsaWdodHMlMjIlMkMlMEEp",highlighted:`>>> from evaluate import evaluator
>>> from datasets import load_dataset
>>> task_evaluator = evaluator("summarization")
>>> data = load_dataset("cnn_dailymail", "3.0.0", split="validation[:40]")
>>> results = task_evaluator.compute(
>>> model_or_pipeline="facebook/bart-large-cnn",
>>> data=data,
>>> input_column="article",
>>> label_column="highlights",
>>> )`,wrap:!1}}),{c(){a=p("p"),a.textContent=m,s=l(),g(o.$$.fragment)},l(t){a=u(t,"P",{"data-svelte-h":!0}),$(a)!=="svelte-kvfsh7"&&(a.textContent=m),s=r(t),h(o.$$.fragment,t)},m(t,w){M(t,a,w),M(t,s,w),f(o,t,w),d=!0},p:C,i(t){d||(v(o.$$.fragment,t),d=!0)},o(t){y(o.$$.fragment,t),d=!1},d(t){t&&(i(a),i(s)),b(o,t)}}}function Wn(T){let a,m="Examples:",s,o,d;return o=new I({props:{code:"ZnJvbSUyMGV2YWx1YXRlJTIwaW1wb3J0JTIwZXZhbHVhdG9yJTBBZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9kYXRhc2V0JTBBdGFza19ldmFsdWF0b3IlMjAlM0QlMjBldmFsdWF0b3IoJTIydHJhbnNsYXRpb24lMjIpJTBBZGF0YSUyMCUzRCUyMGxvYWRfZGF0YXNldCglMjJ3bXQxOSUyMiUyQyUyMCUyMmZyLWRlJTIyJTJDJTIwc3BsaXQlM0QlMjJ2YWxpZGF0aW9uJTVCJTNBNDAlNUQlMjIpJTBBZGF0YSUyMCUzRCUyMGRhdGEubWFwKGxhbWJkYSUyMHglM0ElMjAlN0IlMjJ0ZXh0JTIyJTNBJTIweCU1QiUyMnRyYW5zbGF0aW9uJTIyJTVEJTVCJTIyZGUlMjIlNUQlMkMlMjAlMjJsYWJlbCUyMiUzQSUyMHglNUIlMjJ0cmFuc2xhdGlvbiUyMiU1RCU1QiUyMmZyJTIyJTVEJTdEKSUwQXJlc3VsdHMlMjAlM0QlMjB0YXNrX2V2YWx1YXRvci5jb21wdXRlKCUwQSUyMCUyMCUyMCUyMG1vZGVsX29yX3BpcGVsaW5lJTNEJTIySGVsc2lua2ktTkxQJTJGb3B1cy1tdC1kZS1mciUyMiUyQyUwQSUyMCUyMCUyMCUyMGRhdGElM0RkYXRhJTJDJTBBKQ==",highlighted:`>>> from evaluate import evaluator
>>> from datasets import load_dataset
>>> task_evaluator = evaluator("translation")
>>> data = load_dataset("wmt19", "fr-de", split="validation[:40]")
>>> data = data.map(lambda x: {"text": x["translation"]["de"], "label": x["translation"]["fr"]})
>>> results = task_evaluator.compute(
>>> model_or_pipeline="Helsinki-NLP/opus-mt-de-fr",
>>> data=data,
>>> )`,wrap:!1}}),{c(){a=p("p"),a.textContent=m,s=l(),g(o.$$.fragment)},l(t){a=u(t,"P",{"data-svelte-h":!0}),$(a)!=="svelte-kvfsh7"&&(a.textContent=m),s=r(t),h(o.$$.fragment,t)},m(t,w){M(t,a,w),M(t,s,w),f(o,t,w),d=!0},p:C,i(t){d||(v(o.$$.fragment,t),d=!0)},o(t){y(o.$$.fragment,t),d=!1},d(t){t&&(i(a),i(s)),b(o,t)}}}function Yn(T){let a,m="Examples:",s,o,d;return o=new I({props:{code:"ZnJvbSUyMGV2YWx1YXRlJTIwaW1wb3J0JTIwZXZhbHVhdG9yJTBBZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9kYXRhc2V0JTBBdGFza19ldmFsdWF0b3IlMjAlM0QlMjBldmFsdWF0b3IoJTIyYXV0b21hdGljLXNwZWVjaC1yZWNvZ25pdGlvbiUyMiklMEFkYXRhJTIwJTNEJTIwbG9hZF9kYXRhc2V0KCUyMm1vemlsbGEtZm91bmRhdGlvbiUyRmNvbW1vbl92b2ljZV8xMV8wJTIyJTJDJTIwJTIyZW4lMjIlMkMlMjBzcGxpdCUzRCUyMnZhbGlkYXRpb24lNUIlM0E0MCU1RCUyMiklMEFyZXN1bHRzJTIwJTNEJTIwdGFza19ldmFsdWF0b3IuY29tcHV0ZSglMEElMjAlMjAlMjAlMjBtb2RlbF9vcl9waXBlbGluZSUzRCUyMmh0dHBzJTNBJTJGJTJGaHVnZ2luZ2ZhY2UuY28lMkZvcGVuYWklMkZ3aGlzcGVyLXRpbnkuZW4lMjIlMkMlMEElMjAlMjAlMjAlMjBkYXRhJTNEZGF0YSUyQyUwQSUyMCUyMCUyMCUyMGlucHV0X2NvbHVtbiUzRCUyMnBhdGglMjIlMkMlMEElMjAlMjAlMjAlMjBsYWJlbF9jb2x1bW4lM0QlMjJzZW50ZW5jZSUyMiUyQyUwQSUyMCUyMCUyMCUyMG1ldHJpYyUzRCUyMndlciUyMiUyQyUwQSk=",highlighted:`>>> from evaluate import evaluator
>>> from datasets import load_dataset
>>> task_evaluator = evaluator("automatic-speech-recognition")
>>> data = load_dataset("mozilla-foundation/common_voice_11_0", "en", split="validation[:40]")
>>> results = task_evaluator.compute(
>>> model_or_pipeline="https://huggingface.co/openai/whisper-tiny.en",
>>> data=data,
>>> input_column="path",
>>> label_column="sentence",
>>> metric="wer",
>>> )`,wrap:!1}}),{c(){a=p("p"),a.textContent=m,s=l(),g(o.$$.fragment)},l(t){a=u(t,"P",{"data-svelte-h":!0}),$(a)!=="svelte-kvfsh7"&&(a.textContent=m),s=r(t),h(o.$$.fragment,t)},m(t,w){M(t,a,w),M(t,s,w),f(o,t,w),d=!0},p:C,i(t){d||(v(o.$$.fragment,t),d=!0)},o(t){y(o.$$.fragment,t),d=!1},d(t){t&&(i(a),i(s)),b(o,t)}}}function Vn(T){let a,m='Remember that, in order to process audio files, you need ffmpeg installed (https://ffmpeg.org/download.html)';return{c(){a=p("p"),a.innerHTML=m},l(s){a=u(s,"P",{"data-svelte-h":!0}),$(a)!=="svelte-4bmria"&&(a.innerHTML=m)},m(s,o){M(s,a,o)},p:C,d(s){s&&i(a)}}}function Dn(T){let a,m;return a=new I({props:{code:"ZnJvbSUyMGV2YWx1YXRlJTIwaW1wb3J0JTIwZXZhbHVhdG9yJTBBZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9kYXRhc2V0JTBBJTBBdGFza19ldmFsdWF0b3IlMjAlM0QlMjBldmFsdWF0b3IoJTIyYXVkaW8tY2xhc3NpZmljYXRpb24lMjIpJTBBZGF0YSUyMCUzRCUyMGxvYWRfZGF0YXNldCglMjJzdXBlcmIlMjIlMkMlMjAna3MnJTJDJTIwc3BsaXQlM0QlMjJ0ZXN0JTVCJTNBNDAlNUQlMjIpJTBBcmVzdWx0cyUyMCUzRCUyMHRhc2tfZXZhbHVhdG9yLmNvbXB1dGUoJTBBJTIwJTIwJTIwJTIwbW9kZWxfb3JfcGlwZWxpbmUlM0QlMjIlMjJzdXBlcmIlMkZ3YXYydmVjMi1iYXNlLXN1cGVyYi1rcyUyMiUyMiUyQyUwQSUyMCUyMCUyMCUyMGRhdGElM0RkYXRhJTJDJTBBJTIwJTIwJTIwJTIwbGFiZWxfY29sdW1uJTNEJTIybGFiZWwlMjIlMkMlMEElMjAlMjAlMjAlMjBpbnB1dF9jb2x1bW4lM0QlMjJmaWxlJTIyJTJDJTBBJTIwJTIwJTIwJTIwbWV0cmljJTNEJTIyYWNjdXJhY3klMjIlMkMlMEElMjAlMjAlMjAlMjBsYWJlbF9tYXBwaW5nJTNEJTdCMCUzQSUyMCUyMnllcyUyMiUyQyUyMDElM0ElMjAlMjJubyUyMiUyQyUyMDIlM0ElMjAlMjJ1cCUyMiUyQyUyMDMlM0ElMjAlMjJkb3duJTIyJTdEJTBBKQ==",highlighted:`>>> from evaluate import evaluator
>>> from datasets import load_dataset
>>> task_evaluator = evaluator("audio-classification")
>>> data = load_dataset("superb", 'ks', split="test[:40]")
>>> results = task_evaluator.compute(
>>> model_or_pipeline=""superb/wav2vec2-base-superb-ks"",
>>> data=data,
>>> label_column="label",
>>> input_column="file",
>>> metric="accuracy",
>>> label_mapping={0: "yes", 1: "no", 2: "up", 3: "down"}
>>> )`,wrap:!1}}),{c(){g(a.$$.fragment)},l(s){h(a.$$.fragment,s)},m(s,o){f(a,s,o),m=!0},p:C,i(s){m||(v(a.$$.fragment,s),m=!0)},o(s){y(a.$$.fragment,s),m=!1},d(s){b(a,s)}}}function Bn(T){let a,m=`The evaluator supports raw audio data as well, in the form of a numpy array. However, be aware that calling
the audio column automatically decodes and resamples the audio files, which can be slow for large datasets.`;return{c(){a=p("p"),a.textContent=m},l(s){a=u(s,"P",{"data-svelte-h":!0}),$(a)!=="svelte-y60bpe"&&(a.textContent=m)},m(s,o){M(s,a,o)},p:C,d(s){s&&i(a)}}}function Qn(T){let a,m;return a=new I({props:{code:"ZnJvbSUyMGV2YWx1YXRlJTIwaW1wb3J0JTIwZXZhbHVhdG9yJTBBZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9kYXRhc2V0JTBBJTBBdGFza19ldmFsdWF0b3IlMjAlM0QlMjBldmFsdWF0b3IoJTIyYXVkaW8tY2xhc3NpZmljYXRpb24lMjIpJTBBZGF0YSUyMCUzRCUyMGxvYWRfZGF0YXNldCglMjJzdXBlcmIlMjIlMkMlMjAna3MnJTJDJTIwc3BsaXQlM0QlMjJ0ZXN0JTVCJTNBNDAlNUQlMjIpJTBBZGF0YSUyMCUzRCUyMGRhdGEubWFwKGxhbWJkYSUyMGV4YW1wbGUlM0ElMjAlN0IlMjJhdWRpbyUyMiUzQSUyMGV4YW1wbGUlNUIlMjJhdWRpbyUyMiU1RCU1QiUyMmFycmF5JTIyJTVEJTdEKSUwQXJlc3VsdHMlMjAlM0QlMjB0YXNrX2V2YWx1YXRvci5jb21wdXRlKCUwQSUyMCUyMCUyMCUyMG1vZGVsX29yX3BpcGVsaW5lJTNEJTIyJTIyc3VwZXJiJTJGd2F2MnZlYzItYmFzZS1zdXBlcmIta3MlMjIlMjIlMkMlMEElMjAlMjAlMjAlMjBkYXRhJTNEZGF0YSUyQyUwQSUyMCUyMCUyMCUyMGxhYmVsX2NvbHVtbiUzRCUyMmxhYmVsJTIyJTJDJTBBJTIwJTIwJTIwJTIwaW5wdXRfY29sdW1uJTNEJTIyYXVkaW8lMjIlMkMlMEElMjAlMjAlMjAlMjBtZXRyaWMlM0QlMjJhY2N1cmFjeSUyMiUyQyUwQSUyMCUyMCUyMCUyMGxhYmVsX21hcHBpbmclM0QlN0IwJTNBJTIwJTIyeWVzJTIyJTJDJTIwMSUzQSUyMCUyMm5vJTIyJTJDJTIwMiUzQSUyMCUyMnVwJTIyJTJDJTIwMyUzQSUyMCUyMmRvd24lMjIlN0QlMEEp",highlighted:`>>> from evaluate import evaluator
>>> from datasets import load_dataset
>>> task_evaluator = evaluator("audio-classification")
>>> data = load_dataset("superb", 'ks', split="test[:40]")
>>> data = data.map(lambda example: {"audio": example["audio"]["array"]})
>>> results = task_evaluator.compute(
>>> model_or_pipeline=""superb/wav2vec2-base-superb-ks"",
>>> data=data,
>>> label_column="label",
>>> input_column="audio",
>>> metric="accuracy",
>>> label_mapping={0: "yes", 1: "no", 2: "up", 3: "down"}
>>> )`,wrap:!1}}),{c(){g(a.$$.fragment)},l(s){h(a.$$.fragment,s)},m(s,o){f(a,s,o),m=!0},p:C,i(s){m||(v(a.$$.fragment,s),m=!0)},o(s){y(a.$$.fragment,s),m=!1},d(s){b(a,s)}}}function Sn(T){let a,m,s,o,d,t,w,Eo="The evaluator classes for automatic evaluation.",Na,He,Ga,Le,qo="The main entry point for using the evaluator:",Ra,Y,Oe,cs,Yt,No=`Utility factory method to build an Evaluator.
Evaluators encapsulate a task and a default metric name. They leverage pipeline functionality from transformers
to simplify the evaluation of multiple combinations of models, datasets and metrics for a given task.`,ds,je,Fa,Ke,Go="The base class for all evaluator classes:",za,J,et,ps,Vt,Ro=`The Evaluator class is the class from which all evaluators inherit. Refer to this class for methods shared across
different evaluators.
Base class implementing evaluator operations.`,us,O,tt,ms,Dt,Fo="Ensure the columns required for the evaluation are present in the dataset.",gs,xe,hs,Je,at,fs,Bt,zo="Compute and return metrics.",vs,K,st,ys,Qt,Zo="Infers which split to use if None is given.",bs,Ue,Ms,ee,ot,ws,St,Ao="Load dataset with given subset and split.",$s,Ce,Ts,ke,nt,_s,Pt,Xo="A core method of the Evaluator class, which processes the pipeline outputs for compatibility with the metric.",js,te,lt,xs,Ht,Wo="Prepare data.",Js,Ie,Us,ae,rt,Cs,Lt,Yo="Prepare metric.",ks,Ee,Is,se,it,Es,Ot,Vo="Prepare pipeline.",qs,qe,Za,ct,Aa,dt,Xa,V,pt,Ns,Kt,Do=`Image classification evaluator.
This image classification evaluator can currently be loaded from evaluator() using the default task name
image-classification.
Methods in this class assume a data format compatible with the ImageClassificationPipeline.`,Gs,oe,ut,Rs,ea,Bo="Compute the metric for a given pipeline and dataset combination.",Fs,Ne,Wa,mt,Ya,N,gt,zs,ta,Qo=`Question answering evaluator. This evaluator handles
extractive question answering,
where the answer to the question is extracted from a context.`,Zs,aa,So=`This question answering evaluator can currently be loaded from evaluator() using the default task name
question-answering.`,As,sa,Po=`Methods in this class assume a data format compatible with the
QuestionAnsweringPipeline.`,Xs,F,ht,Ws,oa,Ho="Compute the metric for a given pipeline and dataset combination.",Ys,Ge,Vs,Re,Ds,Fe,Va,ft,Da,D,vt,Bs,na,Lo=`Text classification evaluator.
This text classification evaluator can currently be loaded from evaluator() using the default task name
text-classification or with a "sentiment-analysis" alias.
Methods in this class assume a data format compatible with the TextClassificationPipeline - a single textual
feature as input and a categorical label as output.`,Qs,ne,yt,Ss,la,Oo="Compute the metric for a given pipeline and dataset combination.",Ps,ze,Ba,bt,Qa,G,Mt,Hs,ra,Ko="Token classification evaluator.",Ls,ia,en=`This token classification evaluator can currently be loaded from evaluator() using the default task name
token-classification.`,Os,ca,tn='Methods in this class assume a data format compatible with the TokenClassificationPipeline.',Ks,q,wt,eo,da,an="Compute the metric for a given pipeline and dataset combination.",to,pa,sn='The dataset input and label columns are expected to be formatted as a list of words and a list of labels respectively, following conll2003 dataset. Datasets whose inputs are single strings, and labels are a list of offset are not supported.',ao,Ze,so,Ae,oo,Xe,Sa,$t,Pa,B,Tt,no,ua,on=`Text generation evaluator.
This Text generation evaluator can currently be loaded from evaluator() using the default task name
text-generation.
Methods in this class assume a data format compatible with the TextGenerationPipeline.`,lo,ma,_t,Ha,jt,La,Q,xt,ro,ga,nn=`Text2Text generation evaluator.
This Text2Text generation evaluator can currently be loaded from evaluator() using the default task name
text2text-generation.
Methods in this class assume a data format compatible with the Text2TextGenerationPipeline.`,io,le,Jt,co,ha,ln="Compute the metric for a given pipeline and dataset combination.",po,We,Oa,Ut,Ka,S,Ct,uo,fa,rn=`Text summarization evaluator.
This text summarization evaluator can currently be loaded from evaluator() using the default task name
summarization.
Methods in this class assume a data format compatible with the SummarizationEvaluator.`,mo,re,kt,go,va,cn="Compute the metric for a given pipeline and dataset combination.",ho,Ye,es,It,ts,P,Et,fo,ya,dn=`Translation evaluator.
This translation generation evaluator can currently be loaded from evaluator() using the default task name
translation.
Methods in this class assume a data format compatible with the TranslationPipeline.`,vo,ie,qt,yo,ba,pn="Compute the metric for a given pipeline and dataset combination.",bo,Ve,as,Nt,ss,H,Gt,Mo,Ma,un=`Automatic speech recognition evaluator.
This automatic speech recognition evaluator can currently be loaded from evaluator() using the default task name
automatic-speech-recognition.
Methods in this class assume a data format compatible with the AutomaticSpeechRecognitionPipeline.`,wo,ce,Rt,$o,wa,mn="Compute the metric for a given pipeline and dataset combination.",To,De,os,Ft,ns,L,zt,_o,$a,gn=`Audio classification evaluator.
This audio classification evaluator can currently be loaded from evaluator() using the default task name
audio-classification.
Methods in this class assume a data format compatible with the transformers.AudioClassificationPipeline.`,jo,E,Zt,xo,Ta,hn="Compute the metric for a given pipeline and dataset combination.",Jo,_a,fn="Examples:",Uo,Be,Co,Qe,ko,Se,Io,Pe,ls,At,rs,Ia,is;return d=new W({props:{title:"Evaluator",local:"evaluator",headingTag:"h1"}}),He=new W({props:{title:"Evaluator classes",local:"evaluate.evaluator",headingTag:"h2"}}),Oe=new x({props:{name:"evaluate.evaluator",anchor:"evaluate.evaluator",parameters:[{name:"task",val:": str = None"}],parametersDescription:[{anchor:"evaluate.evaluator.task",description:`task (str) —
The task defining which evaluator will be returned. Currently accepted tasks are:`,name:"task"}],source:"https://github.com/huggingface/evaluate/blob/v0.4.5/src/evaluate/evaluator/__init__.py#L113",returnDescription:`