Clean evidence infer treatment. (#705)

* Clean subset 2.0.

* Clean subset 1.1.

* Minor change to trigger tests.

* Revert "Minor change to trigger tests."

This reverts commit 0a533ee.

Author: stephenbach

promptsource/templates/evidence_infer_treatment/1.1/templates.yaml

@@ -1,127 +1,115 @@
 dataset: evidence_infer_treatment
 subset: '1.1'
 templates:
-  1b538c15-d7b7-4139-8755-fb7d28c19a4d: !Template
+  6148ddc8-b722-4d4f-8498-ff36c45803d6: !Template
     answer_choices: null
-    id: 1b538c15-d7b7-4139-8755-fb7d28c19a4d
+    id: 6148ddc8-b722-4d4f-8498-ff36c45803d6
     jinja: "{% set annotation_length = Prompts.Annotations | length %}\n\n{% set specific_sub_annotation\
       \ = range(0, annotation_length) | choice %}\n\n{% set sub_annotation_length\
       \ = Prompts.Annotations[specific_sub_annotation].Annotations | length %}\n\n\
       {% set sub_sub_annotation = [0] %}\n\n{% if sub_annotation_length > 0 %}\n\n\
-      {{Text[:1200]}} \n\n{{Text[-300:]}}\n\nThe text above contains important details\
-      \ for answering the following questions:\n\nThe relevant annotations:\n\n{{\
-      \ sub_sub_annotation.pop() | replace(0, \"\") }}\n{{ sub_sub_annotation.append(range(0,\
-      \ sub_annotation_length) | choice) | replace(None, \"\") }}\n\n{{Prompts.Annotations[specific_sub_annotation].Annotations[sub_sub_annotation[0]]}}\n\
-      \nNow on the basis of annotation and the text the outcome is:\n\n{% endif %}\n\
-      \n|||\n\n\n{{Prompts.Outcome[specific_sub_annotation]}}"
+      The following text snippets contain important information:\n\n{{Text[:1200]}}\
+      \ \n\n{{Text[-300:]}}\n\nThe relevant annotations are:\n\n{{ sub_sub_annotation.pop()\
+      \ | replace(0, \"\") }}\n{{ sub_sub_annotation.append(range(0, sub_annotation_length)\
+      \ | choice) | replace(None, \"\") }}\n\n{{Prompts.Annotations[specific_sub_annotation].Annotations[sub_sub_annotation[0]]}}\n\
+      \nRegarding the following comparator\n\n{{Prompts.Comparator[specific_sub_annotation]}},\n\
+      \nthe intervention was\n\n{% endif %}\n\n|||\n\n\n{{Prompts.Intervention[specific_sub_annotation]}}.\n"
     metadata: !TemplateMetadata
-      choices_in_prompt: null
-      metrics: []
+      choices_in_prompt: false
+      metrics:
+      - Accuracy
       original_task: false
-    name: template_4
+    name: Identify intervention
     reference: ''
-  7ce46648-2bcc-4e67-95f5-c2a0d0612f9b: !Template
+  9ea1dca5-1867-48f6-9a0f-1c55b19c4606: !Template
     answer_choices: null
-    id: 7ce46648-2bcc-4e67-95f5-c2a0d0612f9b
+    id: 9ea1dca5-1867-48f6-9a0f-1c55b19c4606
     jinja: "{% set annotation_length = Prompts.Annotations | length %}\n\n{% set specific_sub_annotation\
       \ = range(0, annotation_length) | choice %}\n\n{% set sub_annotation_length\
       \ = Prompts.Annotations[specific_sub_annotation].Annotations | length %}\n\n\
       {% set sub_sub_annotation = [0] %}\n\n{% if sub_annotation_length > 0 %}\n\n\
-      {{ sub_sub_annotation.pop() | replace(0, \"\") }}\n{{ sub_sub_annotation.append(range(0,\
-      \ sub_annotation_length) | choice) | replace(None, \"\") }}\n\nAfter reading\
-      \ the following text:\n\n{{Text[:1200]}} \n\n{{Text[-300:]}}\n\nThe relevant\
-      \ annotations:\n\n{{Prompts.Annotations[specific_sub_annotation].Annotations[sub_sub_annotation[0]]}}\n\
-      \nNow if the comparator is:\n\n{{Prompts.Comparator[specific_sub_annotation]}}.\n\
-      \nand the intervention is:\n\n{{Prompts.Intervention[specific_sub_annotation]}}.\n\
-      \n The outcome is: \n\n{% endif %}\n\n|||\n\n{{Prompts.Outcome[specific_sub_annotation]}}"
+      The first text snippet that is important to understand is:\n\n{{Text[:1200]}}\
+      \ \n\nthe second text snippet is:\n\n{{Text[-300:]}}\n\nThe relevant annotations:\n\
+      \n{{ sub_sub_annotation.pop() | replace(0, \"\") }}\n{{ sub_sub_annotation.append(range(0,\
+      \ sub_annotation_length) | choice) | replace(None, \"\") }}\n\n{{Prompts.Annotations[specific_sub_annotation].Annotations[sub_sub_annotation[0]]}}\n\
+      \nRegarding the intervention\n\n{{Prompts.Intervention[specific_sub_annotation]}}\n\
+      \nwith the outcome\n\n{{Prompts.Outcome[specific_sub_annotation]}},\n\nthe comparator\
+      \ was:\n\n{% endif %}\n\n|||\n\n{{Prompts.Comparator[specific_sub_annotation]}}."
     metadata: !TemplateMetadata
-      choices_in_prompt: null
-      metrics: []
+      choices_in_prompt: false
+      metrics:
+      - Accuracy
       original_task: false
-    name: template_2
-    reference: ''
-  7d618260-32fb-405d-ab79-cec67f589de7: !Template
-    answer_choices: null
-    id: 7d618260-32fb-405d-ab79-cec67f589de7
-    jinja: "{% set annotation_length = Prompts.Annotations | length %}\n\n{% set specific_sub_annotation\
-      \ = range(0, annotation_length) | choice %}\n\n{% set sub_annotation_length\
-      \ = Prompts.Annotations[specific_sub_annotation].Annotations | length %}\n\n\
-      {% set sub_sub_annotation = [0] %}\n\n{% if sub_annotation_length > 0 %}\n\n\
-      Read the following text:\n\n{{ sub_sub_annotation.pop() | replace(0, \"\") }}\n\
-      {{ sub_sub_annotation.append(range(0, sub_annotation_length) | choice) | replace(None,\
-      \ \"\") }}\n\n{{Text[:1200]}} \n\n{{Text[-300:]}}\n\nNow the comparator is:\n\
-      \n{{Prompts.Comparator[specific_sub_annotation]}}.\n\nThe intervention is:\n\
-      \n{{Prompts.Intervention[specific_sub_annotation]}}.\n\nThe outcome:\n\n{{Prompts.Outcome[specific_sub_annotation]}}\n\
-      \nis either {{\"significantly increased\"}}, {{\"significantly decreased\"}}\
-      \ or {{\"no significant difference\"}}. Which is it?\n\n{% endif %}\n\n|||\n\
-      \n{% if sub_annotation_length > 0 %}\n\n{{Prompts.Annotations[specific_sub_annotation].Label[sub_sub_annotation[0]]}}\n\
-      \n{% endif %}"
-    metadata: !TemplateMetadata
-      choices_in_prompt: null
-      metrics: []
-      original_task: true
-    name: template_3
+    name: Identify comparator
     reference: ''
-  c999469a-20e0-4c10-a707-3c057d5c0245: !Template
-    answer_choices: null
-    id: c999469a-20e0-4c10-a707-3c057d5c0245
+  bf430e30-a6a4-4bc0-a304-bbc1a06e23fd: !Template
+    answer_choices: significantly increased ||| significantly decreased ||| no significant
+      difference
+    id: bf430e30-a6a4-4bc0-a304-bbc1a06e23fd
     jinja: "{% set annotation_length = Prompts.Annotations | length %}\n\n{% set specific_sub_annotation\
       \ = range(0, annotation_length) | choice %}\n\n{% set sub_annotation_length\
       \ = Prompts.Annotations[specific_sub_annotation].Annotations | length %}\n\n\
       {% set sub_sub_annotation = [0] %}\n\n{% if sub_annotation_length > 0 %}\n\n\
-      The following text snippets contain important information:\n\n{{Text[:1200]}}\
-      \ \n\n{{Text[-300:]}}\n\nThe relevant annotations are:\n\n{{ sub_sub_annotation.pop()\
+      The information required to understand the outcome is below:\n\n{{Text[:1200]}}\
+      \ \n\n{{Text[-300:]}}\n\nThe relevant annotations:\n\n{{ sub_sub_annotation.pop()\
       \ | replace(0, \"\") }}\n{{ sub_sub_annotation.append(range(0, sub_annotation_length)\
       \ | choice) | replace(None, \"\") }}\n\n{{Prompts.Annotations[specific_sub_annotation].Annotations[sub_sub_annotation[0]]}}\n\
-      \nNow if the comparator is:\n\n{{Prompts.Comparator[specific_sub_annotation]}}.\n\
-      \nThe intervention will be:\n\n{% endif %}\n\n|||\n\n\n{{Prompts.Intervention[specific_sub_annotation]}}.\n"
+      \nConsider the intervention\n\n{{Prompts.Intervention[specific_sub_annotation]}}\n\
+      \nwith respect to the comparator\n\n{{Prompts.Comparator[specific_sub_annotation]}}.\n\
+      \nThe outcome\n\n{{Prompts.Outcome[specific_sub_annotation]}}\n\nis either {{\"\
+      significantly increased\"}}, {{\"significantly decreased\"}} or {{\"no significant\
+      \ difference\"}}. Which is it?\n\n{% endif %}\n\n|||\n\n{% if sub_annotation_length\
+      \ > 0 %}\n\n{{Prompts.Annotations[specific_sub_annotation].Label[sub_sub_annotation[0]]}}\n\
+      \n{% endif %}"
     metadata: !TemplateMetadata
-      choices_in_prompt: null
-      metrics: []
-      original_task: false
-    name: template_1
-    reference: ''
-  da67a99f-0472-4658-a410-afe260749d90: !Template
-    answer_choices: null
-    id: da67a99f-0472-4658-a410-afe260749d90
+      choices_in_prompt: true
+      metrics:
+      - Accuracy
+      original_task: true
+    name: Classify outcome with all info
+    reference: Template with the task definition
+  d5fea159-0593-4e99-bb3d-27e5ff1411f9: !Template
+    answer_choices: significantly increased ||| significantly decreased ||| no significant
+      difference
+    id: d5fea159-0593-4e99-bb3d-27e5ff1411f9
     jinja: "{% set annotation_length = Prompts.Annotations | length %}\n\n{% set specific_sub_annotation\
       \ = range(0, annotation_length) | choice %}\n\n{% set sub_annotation_length\
       \ = Prompts.Annotations[specific_sub_annotation].Annotations | length %}\n\n\
       {% set sub_sub_annotation = [0] %}\n\n{% if sub_annotation_length > 0 %}\n\n\
-      The information required to understand the outcome is below:\n\n{{Text[:1200]}}\
-      \ \n\n{{Text[-300:]}}\n\nThe relevant annotations:\n\n{{ sub_sub_annotation.pop()\
-      \ | replace(0, \"\") }}\n{{ sub_sub_annotation.append(range(0, sub_annotation_length)\
-      \ | choice) | replace(None, \"\") }}\n\n{{Prompts.Annotations[specific_sub_annotation].Annotations[sub_sub_annotation[0]]}}\n\
-      \nThe comparator is:\n\n{{Prompts.Comparator[specific_sub_annotation]}}.\n\n\
-      The intervention is:\n\n{{Prompts.Intervention[specific_sub_annotation]}}.\n\
-      \nThe outcome:\n\n{{Prompts.Outcome[specific_sub_annotation]}}\n\nis either\
-      \ {{\"significantly increased\"}}, {{\"significantly decreased\"}} or {{\"no\
-      \ significant difference\"}}. Which is it?\n\n{% endif %}\n\n|||\n\n{% if sub_annotation_length\
-      \ > 0 %}\n\n{{Prompts.Annotations[specific_sub_annotation].Label[sub_sub_annotation[0]]}}\n\
+      Read the following text:\n\n{{ sub_sub_annotation.pop() | replace(0, \"\") }}\n\
+      {{ sub_sub_annotation.append(range(0, sub_annotation_length) | choice) | replace(None,\
+      \ \"\") }}\n\n{{Text[:1200]}} \n\n{{Text[-300:]}}\n\nConsider the intervention\n\
+      \n{{Prompts.Intervention[specific_sub_annotation]}}\n\nwith respect to the comparator\n\
+      \n{{Prompts.Comparator[specific_sub_annotation]}}.\n\nThe outcome\n\n{{Prompts.Outcome[specific_sub_annotation]}}\n\
+      \nis either {{\"significantly increased\"}}, {{\"significantly decreased\"}}\
+      \ or {{\"no significant difference\"}}. Which is it?\n\n{% endif %}\n\n|||\n\
+      \n{% if sub_annotation_length > 0 %}\n\n{{Prompts.Annotations[specific_sub_annotation].Label[sub_sub_annotation[0]]}}\n\
       \n{% endif %}"
     metadata: !TemplateMetadata
-      choices_in_prompt: null
-      metrics: []
+      choices_in_prompt: true
+      metrics:
+      - Accuracy
       original_task: true
-    name: template_with_all_info
-    reference: Template with the task definition
-  fbf5600f-5e70-4c15-9608-f53cec32825f: !Template
+    name: Classify outcome
+    reference: ''
+  fed6ea12-8b97-491b-8741-b05d662454de: !Template
     answer_choices: null
-    id: fbf5600f-5e70-4c15-9608-f53cec32825f
+    id: fed6ea12-8b97-491b-8741-b05d662454de
     jinja: "{% set annotation_length = Prompts.Annotations | length %}\n\n{% set specific_sub_annotation\
       \ = range(0, annotation_length) | choice %}\n\n{% set sub_annotation_length\
       \ = Prompts.Annotations[specific_sub_annotation].Annotations | length %}\n\n\
       {% set sub_sub_annotation = [0] %}\n\n{% if sub_annotation_length > 0 %}\n\n\
-      The first text snippet that is important to understand is:\n\n{{Text[:1200]}}\
-      \ \n\nthe second text snippet is:\n\n{{Text[-300:]}}\n\nThe relevant annotations:\n\
-      \n{{ sub_sub_annotation.pop() | replace(0, \"\") }}\n{{ sub_sub_annotation.append(range(0,\
-      \ sub_annotation_length) | choice) | replace(None, \"\") }}\n\n{{Prompts.Annotations[specific_sub_annotation].Annotations[sub_sub_annotation[0]]}}\n\
-      \nThe intervention is:\n\n{{Prompts.Intervention[specific_sub_annotation]}}.\n\
-      \nThe outcome:\n\n{{Prompts.Outcome[specific_sub_annotation]}}\n\nThe comparator\
-      \ is:\n\n{% endif %}\n\n|||\n\n{{Prompts.Comparator[specific_sub_annotation]}}."
+      {{ sub_sub_annotation.pop() | replace(0, \"\") }}\n{{ sub_sub_annotation.append(range(0,\
+      \ sub_annotation_length) | choice) | replace(None, \"\") }}\n\nAfter reading\
+      \ the following text:\n\n{{Text[:1200]}} \n\n{{Text[-300:]}}\n\nThe relevant\
+      \ annotations:\n\n{{Prompts.Annotations[specific_sub_annotation].Annotations[sub_sub_annotation[0]]}}\n\
+      \nRegarding the comparator\n\n{{Prompts.Comparator[specific_sub_annotation]}}\n\
+      \nand the intervention\n\n{{Prompts.Intervention[specific_sub_annotation]}},\n\
+      \nthe outcome was\n\n{% endif %}\n\n|||\n\n{{Prompts.Outcome[specific_sub_annotation]}}"
     metadata: !TemplateMetadata
-      choices_in_prompt: null
-      metrics: []
+      choices_in_prompt: false
+      metrics:
+      - Accuracy
       original_task: false
-    name: template_5
+    name: Identify outcome
     reference: ''