Add xwinograd

promptsource/templates.py

@@ -27,7 +27,18 @@
 # These are users whose datasets should be included in the results returned by
 # filter_english_datasets (regardless of their metadata)
 
-INCLUDED_USERS = {"Zaid", "craffel", "GEM", "aps", "khalidalt", "shanya", "rbawden", "BigScienceBiasEval", "gsarti"}
+INCLUDED_USERS = {
+    "Zaid",
+    "craffel",
+    "GEM",
+    "aps",
+    "khalidalt",
+    "shanya",
+    "rbawden",
+    "BigScienceBiasEval",
+    "gsarti",
+    "Muennighoff",
+}
 
 # These are the metrics with which templates can be tagged
 METRICS = {

promptsource/templates/Muennighoff/xwinograd/en/templates.yaml

@@ -0,0 +1,110 @@
+dataset: Muennighoff/xwinograd
+subset: en
+templates:
+  28d31908-4aee-4545-aff2-7528cbf39197: !Template
+    answer_choices: '{{option1}} ||| {{option2}}'
+    id: 28d31908-4aee-4545-aff2-7528cbf39197
+    jinja: "{{sentence}}\nReplace the _ in the above sentence with the correct option:\
+      \ \n- {{option1}}\n- {{option2}}\n|||\n{% if answer == '1' %} {{option1}} {%\
+      \ else %} {{ option2 }} {% endif %}"
+    metadata: !TemplateMetadata
+      choices_in_prompt: true
+      languages:
+      - en
+      metrics:
+      - Accuracy
+      original_task: true
+    name: Replace
+    reference: ''
+  50ce5113-882f-4a9d-b21d-8d98b4644295: !Template
+    answer_choices: '{{option1}} ||| {{option2}}'
+    id: 50ce5113-882f-4a9d-b21d-8d98b4644295
+    jinja: 'Fill in the _ in the below sentence:
+
+      {{sentence}}
+
+
+      Choices:
+
+      - {{ option1 }}
+
+      - {{ option2 }}
+
+
+      Answer: ||| {% if answer == ''1'' %} {{option1}} {% else %} {{ option2 }} {%
+      endif %}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: true
+      languages:
+      - en
+      metrics:
+      - Accuracy
+      original_task: true
+    name: fill in the blank
+    reference: ''
+  7f0f6d33-25e2-4394-b1f0-49a2a54767aa: !Template
+    answer_choices: True ||| False
+    id: 7f0f6d33-25e2-4394-b1f0-49a2a54767aa
+    jinja: 'The _ in the sentence below refers to {{option1}}. True or False?
+
+      {{sentence}}|||
+
+      {{answer_choices[answer|int - 1]}}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: true
+      languages:
+      - en
+      metrics:
+      - Accuracy
+      original_task: false
+    name: True or False
+    reference: ''
+  80f9679e-7b6c-4ee7-a348-e905ed9aaf9e: !Template
+    answer_choices: '{{ option1 }} ||| {{ option2 }}'
+    id: 80f9679e-7b6c-4ee7-a348-e905ed9aaf9e
+    jinja: '{{ sentence }} In the previous sentence, does _ refer to {{ option1 }}
+      or  {{ option2 }}? ||| {% if answer == ''1'' %} {{option1}} {% else %} {{ option2
+      }} {% endif %}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: true
+      languages:
+      - en
+      metrics:
+      - Accuracy
+      original_task: true
+    name: does underscore refer to
+    reference: ''
+  bd40cf1f-bda2-4757-b1b5-f1a20a3f7202: !Template
+    answer_choices: '{{option1}} ||| {{option2}}'
+    id: bd40cf1f-bda2-4757-b1b5-f1a20a3f7202
+    jinja: '{{sentence}}
+
+      What does the _ in the above sentence refer to? {{ option1 }} or {{ option2
+      }}? ||| {% if answer == ''1'' %} {{option1}} {% else %} {{ option2 }} {% endif
+      %}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: true
+      languages:
+      - en
+      metrics:
+      - Accuracy
+      original_task: true
+    name: underscore refer to
+    reference: ''
+  ec365d5d-bb5c-488c-93a0-4f90e6011c5d: !Template
+    answer_choices: '{{option1}} ||| {{option2}}'
+    id: ec365d5d-bb5c-488c-93a0-4f90e6011c5d
+    jinja: 'In the sentence below, does the _ stand for {{answer_choices[0]}} or {{answer_choices[1]}}?
+
+      {{sentence}}|||
+
+      {{answer_choices[answer | int - 1]}}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: true
+      languages:
+      - en
+      metrics:
+      - Accuracy
+      original_task: true
+    name: stand for
+    reference: ''

promptsource/templates/Muennighoff/xwinograd/fr/templates.yaml

@@ -0,0 +1,110 @@
+dataset: Muennighoff/xwinograd
+subset: en
+templates:
+  38d31908-4aee-4545-aff2-7528cbf39197: !Template
+    answer_choices: '{{option1}} ||| {{option2}}'
+    id: 38d31908-4aee-4545-aff2-7528cbf39197
+    jinja: "{{sentence}}\nReplace the _ in the above sentence with the correct option:\
+      \ \n- {{option1}}\n- {{option2}}\n|||\n{% if answer == '1' %} {{option1}} {%\
+      \ else %} {{ option2 }} {% endif %}"
+    metadata: !TemplateMetadata
+      choices_in_prompt: true
+      languages:
+      - en
+      metrics:
+      - Accuracy
+      original_task: true
+    name: Replace
+    reference: ''
+  60ce5113-882f-4a9d-b21d-8d98b4644295: !Template
+    answer_choices: '{{option1}} ||| {{option2}}'
+    id: 60ce5113-882f-4a9d-b21d-8d98b4644295
+    jinja: 'Fill in the _ in the below sentence:
+
+      {{sentence}}
+
+
+      Choices:
+
+      - {{ option1 }}
+
+      - {{ option2 }}
+
+
+      Answer: ||| {% if answer == ''1'' %} {{option1}} {% else %} {{ option2 }} {%
+      endif %}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: true
+      languages:
+      - en
+      metrics:
+      - Accuracy
+      original_task: true
+    name: fill in the blank
+    reference: ''
+  8f0f6d33-25e2-4394-b1f0-49a2a54767aa: !Template
+    answer_choices: True ||| False
+    id: 8f0f6d33-25e2-4394-b1f0-49a2a54767aa
+    jinja: 'The _ in the sentence below refers to {{option1}}. True or False?
+
+      {{sentence}}|||
+
+      {{answer_choices[answer|int - 1]}}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: true
+      languages:
+      - en
+      metrics:
+      - Accuracy
+      original_task: false
+    name: True or False
+    reference: ''
+  90f9679e-7b6c-4ee7-a348-e905ed9aaf9e: !Template
+    answer_choices: '{{ option1 }} ||| {{ option2 }}'
+    id: 90f9679e-7b6c-4ee7-a348-e905ed9aaf9e
+    jinja: '{{ sentence }} In the previous sentence, does _ refer to {{ option1 }}
+      or  {{ option2 }}? ||| {% if answer == ''1'' %} {{option1}} {% else %} {{ option2
+      }} {% endif %}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: true
+      languages:
+      - en
+      metrics:
+      - Accuracy
+      original_task: true
+    name: does underscore refer to
+    reference: ''
+  cd40cf1f-bda2-4757-b1b5-f1a20a3f7202: !Template
+    answer_choices: '{{option1}} ||| {{option2}}'
+    id: cd40cf1f-bda2-4757-b1b5-f1a20a3f7202
+    jinja: '{{sentence}}
+
+      What does the _ in the above sentence refer to? {{ option1 }} or {{ option2
+      }}? ||| {% if answer == ''1'' %} {{option1}} {% else %} {{ option2 }} {% endif
+      %}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: true
+      languages:
+      - en
+      metrics:
+      - Accuracy
+      original_task: true
+    name: underscore refer to
+    reference: ''
+  fc365d5d-bb5c-488c-93a0-4f90e6011c5d: !Template
+    answer_choices: '{{option1}} ||| {{option2}}'
+    id: fc365d5d-bb5c-488c-93a0-4f90e6011c5d
+    jinja: 'In the sentence below, does the _ stand for {{answer_choices[0]}} or {{answer_choices[1]}}?
+
+      {{sentence}}|||
+
+      {{answer_choices[answer | int - 1]}}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: true
+      languages:
+      - en
+      metrics:
+      - Accuracy
+      original_task: true
+    name: stand for
+    reference: ''

promptsource/templates/Muennighoff/xwinograd/pt/templates.yaml

@@ -0,0 +1,110 @@
+dataset: Muennighoff/xwinograd
+subset: en
+templates:
+  38d31908-4aee-4545-aff2-7528cbf39197: !Template
+    answer_choices: '{{option1}} ||| {{option2}}'
+    id: 38d31908-4aee-4545-aff2-7528cbf39197
+    jinja: "{{sentence}}\nReplace the _ in the above sentence with the correct option:\
+      \ \n- {{option1}}\n- {{option2}}\n|||\n{% if answer == '1' %} {{option1}} {%\
+      \ else %} {{ option2 }} {% endif %}"
+    metadata: !TemplateMetadata
+      choices_in_prompt: true
+      languages:
+      - en
+      metrics:
+      - Accuracy
+      original_task: true
+    name: Replace
+    reference: ''
+  60ce5113-882f-4a9d-b21d-8d98b4644295: !Template
+    answer_choices: '{{option1}} ||| {{option2}}'
+    id: 60ce5113-882f-4a9d-b21d-8d98b4644295
+    jinja: 'Fill in the _ in the below sentence:
+
+      {{sentence}}
+
+
+      Choices:
+
+      - {{ option1 }}
+
+      - {{ option2 }}
+
+
+      Answer: ||| {% if answer == ''1'' %} {{option1}} {% else %} {{ option2 }} {%
+      endif %}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: true
+      languages:
+      - en
+      metrics:
+      - Accuracy
+      original_task: true
+    name: fill in the blank
+    reference: ''
+  8f0f6d33-25e2-4394-b1f0-49a2a54767aa: !Template
+    answer_choices: True ||| False
+    id: 8f0f6d33-25e2-4394-b1f0-49a2a54767aa
+    jinja: 'The _ in the sentence below refers to {{option1}}. True or False?
+
+      {{sentence}}|||
+
+      {{answer_choices[answer|int - 1]}}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: true
+      languages:
+      - en
+      metrics:
+      - Accuracy
+      original_task: false
+    name: True or False
+    reference: ''
+  90f9679e-7b6c-4ee7-a348-e905ed9aaf9e: !Template
+    answer_choices: '{{ option1 }} ||| {{ option2 }}'
+    id: 90f9679e-7b6c-4ee7-a348-e905ed9aaf9e
+    jinja: '{{ sentence }} In the previous sentence, does _ refer to {{ option1 }}
+      or  {{ option2 }}? ||| {% if answer == ''1'' %} {{option1}} {% else %} {{ option2
+      }} {% endif %}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: true
+      languages:
+      - en
+      metrics:
+      - Accuracy
+      original_task: true
+    name: does underscore refer to
+    reference: ''
+  cd40cf1f-bda2-4757-b1b5-f1a20a3f7202: !Template
+    answer_choices: '{{option1}} ||| {{option2}}'
+    id: cd40cf1f-bda2-4757-b1b5-f1a20a3f7202
+    jinja: '{{sentence}}
+
+      What does the _ in the above sentence refer to? {{ option1 }} or {{ option2
+      }}? ||| {% if answer == ''1'' %} {{option1}} {% else %} {{ option2 }} {% endif
+      %}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: true
+      languages:
+      - en
+      metrics:
+      - Accuracy
+      original_task: true
+    name: underscore refer to
+    reference: ''
+  fc365d5d-bb5c-488c-93a0-4f90e6011c5d: !Template
+    answer_choices: '{{option1}} ||| {{option2}}'
+    id: fc365d5d-bb5c-488c-93a0-4f90e6011c5d
+    jinja: 'In the sentence below, does the _ stand for {{answer_choices[0]}} or {{answer_choices[1]}}?
+
+      {{sentence}}|||
+
+      {{answer_choices[answer | int - 1]}}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: true
+      languages:
+      - en
+      metrics:
+      - Accuracy
+      original_task: true
+    name: stand for
+    reference: ''