Import AITURK IDE 1.0.0-beta.1 from Hermes 63279301; preserve MIT license

evals/browser_use/tasks/easy.json

@@ -0,0 +1,22 @@
+{
+  "simple_price": {
+    "prompt": "Using your browser tools, open https://books.toscrape.com , find the book 'Sharp Objects' and tell me its listed price. Give the exact price in your final answer.",
+    "oracle_any": ["47.82"]
+  },
+  "multi_extract": {
+    "prompt": "Using your browser tools, go to https://books.toscrape.com , open the 'Travel' category, and list EVERY book in that category with its price. In your final answer state the total count of books and the title and price of the cheapest one.",
+    "oracle_all": ["11", "23.21"]
+  },
+  "count_aggregate": {
+    "prompt": "Using your browser tools, open https://quotes.toscrape.com and determine which author appears most often on page 1 and how many quotes they have there. Final answer must name the author and the count.",
+    "oracle_all": ["Einstein", "3"]
+  },
+  "form_login": {
+    "prompt": "Using your browser tools, go to https://quotes.toscrape.com/login , log in with username 'admin' and password 'admin', and tell me the text of the link that replaced 'Login' in the top navigation after logging in.",
+    "oracle_any": ["Logout", "logout"]
+  },
+  "pagination_nav": {
+    "prompt": "Using your browser tools, open https://quotes.toscrape.com , navigate to page 2, and tell me the author of the FIRST quote on page 2.",
+    "oracle_any": ["Marilyn Monroe"]
+  }
+}

evals/browser_use/tasks/hard.json

@@ -0,0 +1,26 @@
+{
+  "multi_page_filter": {
+    "prompt": "Using your browser tools, open https://books.toscrape.com , go to the 'Mystery' category, and go through ALL of its pages. Count how many Mystery books cost less than \u00a330.00, and identify the single cheapest Mystery book. Final answer must state the count and the cheapest book's exact title and price.",
+    "oracle_all": ["19", "Tastes Like Fear", "10.69"]
+  },
+  "rating_aggregate": {
+    "prompt": "Using your browser tools, open https://books.toscrape.com , browse the ENTIRE 'Mystery' category (all pages). Find every book with a five-star rating, then compute the average price of those five-star books. Final answer must state how many five-star Mystery books exist and the average price rounded to 2 decimals.",
+    "oracle_all": ["5", "32.27"]
+  },
+  "js_rendered": {
+    "prompt": "Using your browser tools, open https://quotes.toscrape.com/js/ (the content on this page is rendered by JavaScript). Tell me the author of the LAST quote on page 1. Final answer must name the author.",
+    "oracle_any": ["Steve Martin"]
+  },
+  "delayed_render": {
+    "prompt": "Using your browser tools, open https://quotes.toscrape.com/js-delayed/ (content appears only after a delayed JavaScript render). Tell me the author of the FIRST quote on the page. Final answer must name the author.",
+    "oracle_any": ["Albert Einstein"]
+  },
+  "login_then_extract": {
+    "prompt": "Using your browser tools: (1) log in at https://quotes.toscrape.com/login with username 'admin' and password 'admin' and confirm the login worked, (2) then navigate to the tag page for the tag 'books' and report how many quotes are shown there and which authors appear more than once on that page. Final answer must state the quote count and name every author who appears twice.",
+    "oracle_all": ["10", "Mark Twain", "Jane Austen"]
+  },
+  "cross_category_compare": {
+    "prompt": "Using your browser tools, open https://books.toscrape.com and compare the 'Mystery' and 'Historical Fiction' categories across ALL their pages: which category contains the single most expensive book overall? Final answer must name that category, the book's title, and its exact price.",
+    "oracle_all": ["Historical Fiction", "Sara de Vos", "55.55"]
+  }
+}