instruction
I wanted to see the full menu of the Hard Rock Cafe in Amsterdam on the OpenTable app.
previous_actions
{"name": "click", "parameters": {"x": 460, "y": 1119}}
{"name": "click", "parameters": {"x": 745, "y": 1228}}
{"name": "click", "parameters": {"x": 540, "y": 2044}}
{"name": "scroll", "parameters": {"direction": "down"}}
{"name": "scroll", "parameters": {"direction": "down"}}
active_application
OpenTable
screen_description
windows {
  bounds_in_screen {
    right: 1080
    bottom: 136
  }
  id: 3
  layer: 1
  window_type: TYPE_SYSTEM
  tree {
    nodes {
      bounds_in_screen {
        right: 1080
        bottom: 136
      }
      class_name: "android.widget.FrameLayout"
    }
    nodes {
      bounds_in_screen {
        right: 1080
        bottom: 136
      }
      class_name: "android.widget.FrameLayout"
      depth: 1
    }
    nodes {
      bounds_in_screen {
        right: 1080
        bottom: 136
      }
      class_name: "android.widget.FrameLayout"
      depth: 1
    }
    nodes {
      bounds_in_screen {
        right: 1080
        bottom: 136
      }
      class_name: "android.widget.FrameLayout"
      depth: 1
    }
    nodes {
      bounds_in_screen {
        right: 1080
        bottom: 136
      }
      class_name: "android.widget.FrameLayout"
      depth: 2
    }
    nodes {
      bounds_in_screen {
        left: 47
        right: 1027
        bottom: 136
      }
      class_name: "android.widget.LinearLayout"
      depth: 3
    }
    nodes {
      bounds_in_screen {
        left: 47
        top: 1
        right: 454
        bottom: 136
      }
      class_name: "android.widget.FrameLayout"
      depth: 4
    }
    nodes {
      bounds_in_screen {
        left: 454
        top: 1
        right: 599
        bottom: 136
      }
      class_name: "android.view.View"
      depth: 4
    }
    nodes {
      bounds_in_screen {
        left: 599
        top: 1
        right: 1006
        bottom: 136
      }
      class_name: "android.widget.FrameLayout"
      depth: 4
    }
    nodes {
      bounds_in_screen {
        left: 47
        top: 1
        right: 318
        bottom: 136
      }
      class_name: "android.widget.FrameLayout"
      depth: 5
    }
    nodes {
      bounds_in_screen {
        left: 924
        top: 1
        right: 1006
        bottom: 136
      }
      class_name: "android.widget.LinearLayout"
      depth: 5
    }
    nodes {
      bounds_in_screen {
        left: 47
        top: 1
        right: 318
        bottom: 136
      }
      class_name: "android.widget.LinearLayout"
      depth: 6
    }
    nodes {
      bounds_in_screen {
        left: 924
        top: 1
        right: 1006
        bottom: 136
      }
      class_name: "android.widget.LinearLayout"
      depth: 6
    }
    nodes {
      bounds_in_screen {
        left: 47
        top: 1
        right: 144
        bottom: 136
      }
      class_name: "android.widget.TextView"
      content_description: "11:52 PM"
      text: "11:52"
      depth: 7
    }
    nodes {
      bounds_in_screen {
        left: 144
        top: 1
        right: 318
        bottom: 136
      }
      class_name: "android.widget.FrameLayout"
      depth: 7
    }
    nodes {
      bounds_in_screen {
        left: 924
        top: 1
        right: 986
        bottom: 136
      }
      class_name: "android.widget.LinearLayout"
      depth: 7
    }
    nodes {
      bounds_in_screen {
        left: 986
        top: 1
        right: 1006
        bottom: 136
      }
      class_name: "android.widget.LinearLayout"
      content_description: "Battery charging, 100 percent."
      depth: 7
    }
    nodes {
      bounds_in_screen {
        left: 144
        top: 1
        right: 318
        bottom: 136
      }
      class_name: "android.view.ViewGroup"
      depth: 8
    }
    nodes {
      bounds_in_screen {
        left: 924
        top: 39
        right: 970
        bottom: 97
      }
      class_name: "android.widget.FrameLayout"
      content_description: "Wifi two bars."
      depth: 8
    }
    nodes {
      bounds_in_screen {
        left: 986
        top: 51
        right: 1006
        bottom: 85
      }
      class_name: "android.widget.ImageView"
      depth: 8
    }
    nodes {
      bounds_in_screen {
        left: 144
        top: 1
        right: 202
        bottom: 136
      }
      class_name: "android.widget.ImageView"
      content_description: "Drive notification: 1 file now available offline"
      depth: 9
    }
    nodes {
      bounds_in_screen {
        left: 202
        top: 1
        right: 260
        bottom: 136
      }
      class_name: "android.widget.ImageView"
      content_description: "AccuWeather notification: "
      depth: 9
    }
    nodes {
      bounds_in_screen {
        left: 260
        top: 1
        right: 318
        bottom: 136
      }
      class_name: "android.widget.ImageView"
      content_description: "Google notification: Tomorrow in Bandhwari: Sunny"
      depth: 9
    }
    nodes {
      bounds_in_screen {
        left: 931
        top: 39
        right: 970
        bottom: 97
      }
      class_name: "android.widget.LinearLayout"
      depth: 9
    }
    nodes {
      bounds_in_screen {
        left: 931
        top: 48
        right: 970
        bottom: 87
      }
      class_name: "android.widget.FrameLayout"
      depth: 10
    }
    nodes {
      bounds_in_screen {
        left: 931
        top: 48
        right: 970
        bottom: 87
      }
      class_name: "android.widget.ImageView"
      depth: 11
    }
  }
}
windows {
  bounds_in_screen {
    right: 1080
    bottom: 2400
  }
  id: 126
  title: "OpenTable"
  window_type: TYPE_APPLICATION
  is_active: true
  is_focused: true
  tree {
    nodes {
      bounds_in_screen {
        right: 1080
        bottom: 2400
      }
      class_name: "android.widget.FrameLayout"
    }
    nodes {
      bounds_in_screen {
        right: 1080
        bottom: 2337
      }
      class_name: "android.widget.LinearLayout"
      depth: 1
    }
    nodes {
      bounds_in_screen {
        right: 1080
        bottom: 136
      }
      class_name: "android.view.View"
      depth: 1
    }
    nodes {
      bounds_in_screen {
        top: 2337
        right: 1080
        bottom: 2400
      }
      class_name: "android.view.View"
      depth: 1
    }
    nodes {
      bounds_in_screen {
        top: 136
        right: 1080
        bottom: 2337
      }
      class_name: "android.widget.FrameLayout"
      depth: 2
    }
    nodes {
      bounds_in_screen {
        top: 136
        right: 1080
        bottom: 2337
      }
      class_name: "android.widget.FrameLayout"
      depth: 3
    }
    nodes {
      bounds_in_screen {
        top: 136
        right: 1080
        bottom: 2337
      }
      class_name: "android.widget.FrameLayout"
      depth: 4
    }
    nodes {
      bounds_in_screen {
        top: 136
        right: 1080
        bottom: 2337
      }
      class_name: "android.widget.FrameLayout"
      depth: 5
    }
    nodes {
      bounds_in_screen {
        top: 136
        right: 1080
        bottom: 2337
      }
      class_name: "android.widget.LinearLayout"
      depth: 6
    }
    nodes {
      bounds_in_screen {
        top: 283
        right: 1080
        bottom: 2337
      }
      class_name: "android.widget.FrameLayout"
      depth: 6
    }
    nodes {
      bounds_in_screen {
        top: 136
        right: 1080
        bottom: 283
      }
      class_name: "android.view.ViewGroup"
      content_description: "Hard Rock Cafe - Amsterdam"
      depth: 7
    }
    nodes {
      bounds_in_screen {
        top: 283
        right: 1080
        bottom: 2337
      }
      class_name: "android.view.View"
      depth: 7
    }
    nodes {
      bounds_in_screen {
        top: 283
        right: 1080
        bottom: 2337
      }
      class_name: "android.view.View"
      depth: 7
    }
    nodes {
      bounds_in_screen {
        top: 283
        right: 1080
        bottom: 2337
      }
      class_name: "androidx.recyclerview.widget.RecyclerView"
      is_focusable: true
      is_scrollable: true
      depth: 7
    }
    nodes {
      bounds_in_screen {
        top: 136
        right: 147
        bottom: 283
      }
      class_name: "android.widget.ImageButton"
      content_description: "Navigate up"
      is_clickable: true
      is_focusable: true
      depth: 8
    }
    nodes {
      bounds_in_screen {
        left: 157
        top: 178
        right: 772
        bottom: 241
      }
      class_name: "android.widget.TextView"
      text: "Hard Rock Cafe - Amsterdam"
      depth: 8
    }
    nodes {
      bounds_in_screen {
        left: 1080
        top: 136
        right: 1080
        bottom: 283
      }
      class_name: "androidx.appcompat.widget.LinearLayoutCompat"
      depth: 8
    }
    nodes {
      bounds_in_screen {
        top: 283
        right: 1080
        bottom: 318
      }
      class_name: "android.widget.LinearLayout"
      depth: 8
    }
    nodes {
      bounds_in_screen {
        top: 318
        right: 1080
        bottom: 597
      }
      class_name: "android.widget.LinearLayout"
      depth: 8
    }
    nodes {
      bounds_in_screen {
        top: 597
        right: 1080
        bottom: 760
      }
      class_name: "android.widget.LinearLayout"
      depth: 8
    }
    nodes {
      bounds_in_screen {
        top: 760
        right: 1080
        bottom: 1082
      }
      class_name: "android.widget.LinearLayout"
      depth: 8
    }
    nodes {
      bounds_in_screen {
        top: 1082
        right: 1080
        bottom: 1447
      }
      class_name: "android.widget.LinearLayout"
      depth: 8
    }
    nodes {
      bounds_in_screen {
        top: 1447
        right: 1080
        bottom: 1769
      }
      class_name: "android.widget.LinearLayout"
      depth: 8
    }
    nodes {
      bounds_in_screen {
        top: 1769
        right: 1080
        bottom: 2091
      }
      class_name: "android.widget.LinearLayout"
      depth: 8
    }
    nodes {
      bounds_in_screen {
        top: 2091
        right: 1080
        bottom: 2254
      }
      class_name: "android.widget.LinearLayout"
      depth: 8
    }
    nodes {
      bounds_in_screen {
        top: 2254
        right: 1080
        bottom: 2337
      }
      class_name: "android.widget.LinearLayout"
      depth: 8
    }
    nodes {
      bounds_in_screen {
        left: 42
        top: 283
        right: 1038
        bottom: 275
      }
      class_name: "android.widget.RelativeLayout"
      depth: 9
    }
    nodes {
      bounds_in_screen {
        top: 317
        right: 1080
        bottom: 318
      }
      class_name: "android.view.View"
      depth: 9
    }
    nodes {
      bounds_in_screen {
        left: 42
        top: 360
        right: 1038
        bottom: 554
      }
      class_name: "android.widget.RelativeLayout"
      depth: 9
    }
    nodes {
      bounds_in_screen {
        top: 596
        right: 1080
        bottom: 597
      }
      class_name: "android.view.View"
      depth: 9
    }
    nodes {
      bounds_in_screen {
        top: 597
        right: 1080
        bottom: 600
      }
      class_name: "android.view.View"
      depth: 9
    }
    nodes {
      bounds_in_screen {
        top: 650
        right: 1080
        bottom: 707
      }
      class_name: "android.widget.TextView"
      text: "SALADS & BOWLS"
      is_focusable: true
      depth: 9
    }
    nodes {
      bounds_in_screen {
        top: 757
        right: 1080
        bottom: 760
      }
      class_name: "android.view.View"
      depth: 9
    }
    nodes {
      bounds_in_screen {
        left: 42
        top: 802
        right: 1038
        bottom: 1039
      }
      class_name: "android.widget.RelativeLayout"
      depth: 9
    }
    nodes {
      bounds_in_screen {
        top: 1081
        right: 1080
        bottom: 1082
      }
      class_name: "android.view.View"
      depth: 9
    }
    nodes {
      bounds_in_screen {
        left: 42
        top: 1124
        right: 1038
        bottom: 1404
      }
      class_name: "android.widget.RelativeLayout"
      depth: 9
    }
    nodes {
      bounds_in_screen {
        top: 1446
        right: 1080
        bottom: 1447
      }
      class_name: "android.view.View"
      depth: 9
    }
    nodes {
      bounds_in_screen {
        left: 42
        top: 1489
        right: 1038
        bottom: 1726
      }
      class_name: "android.widget.RelativeLayout"
      depth: 9
    }
    nodes {
      bounds_in_screen {
        top: 1768
        right: 1080
        bottom: 1769
      }
      class_name: "android.view.View"
      depth: 9
    }
    nodes {
      bounds_in_screen {
        left: 42
        top: 1811
        right: 1038
        bottom: 2048
      }
      class_name: "android.widget.RelativeLayout"
      depth: 9
    }
    nodes {
      bounds_in_screen {
        top: 2090
        right: 1080
        bottom: 2091
      }
      class_name: "android.view.View"
      depth: 9
    }
    nodes {
      bounds_in_screen {
        top: 2091
        right: 1080
        bottom: 2094
      }
      class_name: "android.view.View"
      depth: 9
    }
    nodes {
      bounds_in_screen {
        top: 2144
        right: 1080
        bottom: 2201
      }
      class_name: "android.widget.TextView"
      text: "SPECIALTY ENTRÉES"
      is_focusable: true
      depth: 9
    }
    nodes {
      bounds_in_screen {
        top: 2251
        right: 1080
        bottom: 2254
      }
      class_name: "android.view.View"
      depth: 9
    }
    nodes {
      bounds_in_screen {
        left: 42
        top: 2296
        right: 1038
        bottom: 2337
      }
      class_name: "android.widget.RelativeLayout"
      depth: 9
    }
    nodes {
      bounds_in_screen {
        top: 2575
        right: 1080
        bottom: 2337
      }
      class_name: "android.view.View"
      depth: 9
    }
    nodes {
      bounds_in_screen {
        left: 42
        top: 283
        right: 564
        bottom: 138
      }
      class_name: "android.widget.TextView"
      text: "FRIED CHICKEN SANDWICH"
      is_focusable: true
      depth: 10
    }
    nodes {
      bounds_in_screen {
        left: 42
        top: 283
        right: 1038
        bottom: 275
      }
      class_name: "android.widget.TextView"
      text: "Crispy buttermilk-marinated chicken breast with leaf lettuce, vine-ripened tomato and ranch dressing, served on a toasted fresh brioche bun."
      is_focusable: true
      depth: 10
    }
    nodes {
      bounds_in_screen {
        left: 42
        top: 360
        right: 626
        bottom: 417
      }
      class_name: "android.widget.TextView"
      text: "BUFFALO CHICKEN SANDWICH"
      is_focusable: true
      depth: 10
    }
    nodes {
      bounds_in_screen {
        left: 42
        top: 417
        right: 1038
        bottom: 554
      }
      class_name: "android.widget.TextView"
      text: "Buttermilk-marinated fried chicken tossed with our classic Buffalo sauce with leaf lettuce, vine-ripened tomato and ranch dressing, served on a toasted fresh brioche bun."
      is_focusable: true
      depth: 10
    }
    nodes {
      bounds_in_screen {
        left: 42
        top: 802
        right: 698
        bottom: 859
      }
      class_name: "android.widget.TextView"
      text: "GRILLED CHICKEN CAESAR SALAD"
      is_focusable: true
      depth: 10
    }
    nodes {
      bounds_in_screen {
        left: 42
        top: 859
        right: 1038
        bottom: 1039
      }
      class_name: "android.widget.TextView"
      text: "Chicken breast, grilled and sliced with fresh romaine tossed in a classic Caesar dressing, topped with parmesan crisps, croutons and shaved parmesan cheese.\nSubstitute Grilled Salmon for an additional cost"
      is_focusable: true
      depth: 10
    }
    nodes {
      bounds_in_screen {
        left: 42
        top: 1124
        right: 685
        bottom: 1181
      }
      class_name: "android.widget.TextView"
      text: "GRILLED SALMON NOODLE BOWL"
      is_focusable: true
      depth: 10
    }
    nodes {
      bounds_in_screen {
        left: 42
        top: 1181
        right: 1038
        bottom: 1404
      }
      class_name: "android.widget.TextView"
      text: "Ramen noodles tossed in a sesame-soy dressing over a bed of mixed greens topped with edamame,\njulienned carrots, red peppers, red cabbage, and green onions, served with grilled salmon. Substitute Grilled Chicken or Grilled Steak for an additional cost."
      is_focusable: true
      depth: 10
    }
    nodes {
      bounds_in_screen {
        left: 42
        top: 1489
        right: 679
        bottom: 1546
      }
      class_name: "android.widget.TextView"
      text: "SOUTHWESTERN CHICKEN BOWL"
      is_focusable: true
      depth: 10
    }
    nodes {
      bounds_in_screen {
        left: 42
        top: 1546
        right: 1038
        bottom: 1726
      }
      class_name: "android.widget.TextView"
      text: "Fajita-spiced chicken, quinoa corn salad, pico de gallo, seasoned black beans and red cabbage served on mixed greens tossed in guacamole ranch dressing. Substitute Grilled Steak for an additional cost."
      is_focusable: true
      depth: 10
    }
    nodes {
      bounds_in_screen {
        left: 42
        top: 1811
        right: 310
        bottom: 1868
      }
      class_name: "android.widget.TextView"
      text: "STEAK SALAD"
      is_focusable: true
      depth: 10
    }
    nodes {
      bounds_in_screen {
        left: 42
        top: 1868
        right: 1038
        bottom: 2048
      }
      class_name: "android.widget.TextView"
      text: "Grilled steak on a bed of fresh mixed greens tossed in a blue cheese vinaigrette, with pickled red onions, red peppers and Roma tomatoes, finished with crispy shoestring onions and blue cheese crumbles."
      is_focusable: true
      depth: 10
    }
    nodes {
      bounds_in_screen {
        left: 42
        top: 2296
        right: 361
        bottom: 2337
      }
      class_name: "android.widget.TextView"
      text: "COWBOY RIBEYE"
      is_focusable: true
      depth: 10
    }
    nodes {
      bounds_in_screen {
        left: 42
        top: 2353
        right: 1038
        bottom: 2337
      }
      class_name: "android.widget.TextView"
      text: "397g grilled ribeye topped with herb butter, served with golden mashed potatoes and fresh vegetables. Enjoy Surf n\' Turf style with One Night in Bangkok Spicy ShrimpTM for an additional cost."
      is_focusable: true
      depth: 10
    }
  }
}

assistant
!FUNCTIONCALL{"name": "scroll", "parameters": {"direction": "down"}}
[{"type": "function", "function": {"name": "click", "description": "Click on a specific location on the screen", "parameters": {"type": "object", "properties": {"x": {"type": "integer", "description": "Coordinate on the x axis to click."}, "y": {"type": "integer", "description": "Coordinate on the y axis to click."}}, "required": ["x", "y"]}}}, {"type": "function", "function": {"name": "long_press", "description": "Long press on a specific location on the screen.", "parameters": {"type": "object", "properties": {"x": {"type": "integer", "description": "Coordinate on the x axis to long press."}, "y": {"type": "integer", "description": "Coordinate on the y axis to long press."}}, "required": ["x", "y"]}}}, {"type": "function", "function": {"name": "input_text", "description": "Write text inside of an already selected text field, typically this is preceeded by a 'click' action that selects a field on the screen to type into.", "parameters": {"type": "object", "properties": {"text": {"type": "string", "description": "The text to input in the field."}}, "required": ["text"]}}}, {"type": "function", "function": {"name": "scroll", "description": "Scroll a fixed amount in a specific direction.", "parameters": {"type": "object", "properties": {"direction": {"type": "string", "enum": ["up", "down", "left", "right"], "description": "The direction to scroll in."}}, "required": ["direction"]}}}, {"type": "function", "function": {"name": "navigate_home", "description": "Navigate to the home screen from the current screen you are on.", "parameters": {"type": "object", "properties": {}}}}, {"type": "function", "function": {"name": "navigate_back", "description": "Navigate back to the previous page.", "parameters": {"type": "object", "properties": {}}}}, {"type": "function", "function": {"name": "open_app", "description": "Open an application.", "parameters": {"type": "object", "properties": {"app_name": {"type": "string", "description": "The name of the application to open."}}, "required": ["app_name"]}}}, {"type": "function", "function": {"name": "wait", "description": "Wait for a fixed amount of time. This is useful when you need to 'await' the results of a page that is still loading.", "parameters": {"type": "object", "properties": {}}}}, {"type": "function", "function": {"name": "status", "description": "Once you have completed the work required to fulfill an instruction, report your status by calling this function with 'successful'. If you find that a task is impossible to complete,  call this with 'infeasible' after finding that information.", "parameters": {"type": "object", "properties": {"goal_status": {"type": "string", "enum": ["successful", "infeasible"], "description": "The status of the completion of a instruction, either 'successful' or 'infeasible', called to conclude an episode."}}, "required": ["goal_status"]}}}]