text: (this.webpackJsonpdashboard=this.webpackJsonpdashboard||[]).push([[158],{143:function(e,t,a){},144:function(e,t,a){},146:function(e,t,a){},148:function(e,t,a){},149:function(e,t,a){},151:function(e,t,a){},152:function(e,t,a){},153:function(e,t,a){},154:function(e,t,a){},155:function(e,t,a){},203:function(e,t,a){},22:function(e,t,a){"use strict";var n=a(2),i=a(3),s=a(6),r=a(5),o=a(4),l=a(0),c=a.n(l),h=a(45),u=a(10),d=function(e){Object(r.a)(a,e);var t=Object(o.a)(a);function a(e){var i;return Object(n.a)(this,a),(i=t.call(this,e)).onResize=i.onResize.bind(Object(s.a)(i)),i.initialState={height:e.height,width:e.width,rgb:null,depth:null},i.state=i.initialState,i}return Object(i.a)(a,[{key:"onResize",value:function(e,t,a,n,i){this.setState({width:parseInt(a.style.width,10),height:parseInt(a.style.height,10)})}},{key:"componentDidMount",value:function(){this.props.stateManager&&this.props.stateManager.connect(this)}},{key:"componentWillUnmount",value:function(){this.props.stateManager&&this.props.stateManager.disconnect(this)}},{key:"render",value:function(){var e=this.state,t=e.height,a=e.width,n=e.rgb,i=e.depth,s=this.props,r=s.offsetW,o=s.offsetH,l=s.isMobile,d=n;return null===n&&null==i?l?c.a.createElement("p",null,"Loading..."):c.a.createElement(h.a,{default:{x:r,y:o,width:a,height:t},lockAspectRatio:!0,onResize:this.onResize},c.a.createElement("p",null,"Loading...")):("depth"===this.props.type&&(d=i),l?c.a.createElement(u.Stage,{width:a,height:t},c.a.createElement(u.Layer,null,c.a.createElement(u.Image,{image:d,width:a,height:t}))):c.a.createElement(h.a,{default:{x:r,y:o,width:a,height:t},lockAspectRatio:!0,onResize:this.onResize},c.a.createElement(u.Stage,{width:a,height:t},c.a.createElement(u.Layer,null,c.a.createElement(u.Image,{image:d,width:a,height:t})))))}}]),a}(c.a.Component);t.a=d},34:function(e,t,a){"use strict";var n=a(17),i=a(2),s=a(3),r=a(6),o=a(5),l=a(4),c=a(0),h=a.n(c),u=a(45),d=a(10),p=a(48),m=["rgba(0,200,0,.5)","rgba(200,0,0,.5)","rgba(0,100,255,.5)","rgba(255,150,0,.5)","rgba(100,255,200,.5)","rgba(200,200,0,.5)","rgba(0,200,150,.5)","rgba(200,0,200,.5)","rgba(0,204,255,.5)"],g=function(e){Object(o.a)(a,e);var t=Object(l.a)(a);function a(e){var n;return Object(i.a)(this,a),(n=t.call(this,e)).addObject=n.addObject.bind(Object(r.a)(n)),n.onResize=n.onResize.bind(Object(r.a)(n)),n.onFixup=n.onFixup.bind(Object(r.a)(n)),n.onAnnotationSave=n.onAnnotationSave.bind(Object(r.a)(n)),n.onModelSwitch=n.onModelSwitch.bind(Object(r.a)(n)),n.onPrevFrame=n.onPrevFrame.bind(Object(r.a)(n)),n.onNextFrame=n.onNextFrame.bind(Object(r.a)(n)),n.initialState={height:e.height,width:e.width,scale:1,rgb:null,objects:null,modelMetrics:null,offline:!1,updateFixup:!1},n.state=n.initialState,n}return Object(s.a)(a,[{key:"componentDidUpdate",value:function(){this.state.updateFixup&&(this.onFixup(),this.setState({updateFixup:!1}))}},{key:"addObject",value:function(e){var t=this.state.objects?this.state.objects.concat(e):[e];this.setState({objects:t})}},{key:"onResize",value:function(e,t,a,n,i){this.setState({width:parseInt(a.style.width,10),height:parseInt(a.style.height,10)})}},{key:"onFixup",value:function(){if(this.props.stateManager){var e=this.props.stateManager,t=void 0;if(e.refs.forEach((function(e){e instanceof p.a&&(t=e)})),void 0!==t){var a=this.state.objects.map((function(e){return{mask:e.mask?e.mask.map((function(e){return e.map((function(e){return{x:e[0]/500,y:e[1]/500}}))})):[],label:e.label,properties:e.properties.split("\n "),type:e.type,id:e.id}}));t.setState({image:this.state.rgb,objects:a});for(var n=e.dashboardLayout,i=0;i<n._getAllContentItems().length;i++)if("ObjectFixup"===n._getAllContentItems()[i].config.component){var s=n._getAllContentItems()[i];s.parent.setActiveContentItem(s)}}}}},{key:"onAnnotationSave",value:function(){this.props.stateManager&&this.props.stateManager.onSave()}},{key:"onModelSwitch",value:function(){this.props.stateManager&&(console.log("switching model..."),this.props.stateManager.socket.emit("switch_detector"))}},{key:"onPrevFrame",value:function(){this.props.stateManager&&this.props.stateManager.previousFrame()}},{key:"onNextFrame",value:function(){this.props.stateManager&&this.props.stateManager.nextFrame()}},{key:"componentDidMount",value:function(){this.props.stateManager&&this.props.stateManager.connect(this)}},{key:"render",value:function(){var e=this,t=this.state,a=t.height,i=t.width,s=t.rgb,r=t.objects,o=this.props,l=o.offsetW,c=o.offsetH;if(null===s)return h.a.createElement(u.a,{default:{x:l,y:c,width:i,height:a},lockAspectRatio:!0,onResize:this.onResize},h.a.createElement("p",null,"Loading..."));var p=[],g=r;null===r&&(g=[]);var f=0;g.forEach((function(t,a){if("person"!==t.label){var i=t.id,s=a;Number.isInteger(i)&&(s=i);var r=String(i).concat("_"+t.label),o=(t.properties,m[s%m.length]),l=e.state.scale,c=parseInt(t.bbox[0]*l),u=parseInt(t.bbox[1]*l),g=parseInt(t.bbox[2]*l),v=parseInt(t.bbox[3]*l)-u,b=g-c;if(p.push(h.a.createElement(d.Rect,{key:f,x:c,y:u,width:b,height:v,fillEnabled:!1,stroke:o,opacity:1})),t&&t.mask)for(var y=function(e){var i=t.mask[e].map((function(e){return[e[0]*l,e[1]*l]}));p.push(h.a.createElement(d.Shape,{sceneFunc:function(e,t){e.beginPath(),e.moveTo.apply(e,Object(n.a)(i[0]));for(var a=1;a<i.length;a++)e.lineTo.apply(e,Object(n.a)(i[a]));e.closePath(),e.fillStrokeShape(t)},fill:o,opacity:1,stroke:"detector"===t.type?"white":"black",strokeWidth:1,key:"".concat(a,"-").concat(e)}))},k=0;k<t.mask.length;k++)y(k);p.push(h.a.createElement(d.Text,{key:[f++,r],text:r,x:c,y:u,fill:"white",opacity:1,fontSize:20}))}}));var v=null;return this.state.offline&&(v=h.a.createElement("span",{style:{float:"right"}},h.a.createElement("button",{onClick:this.onPrevFrame},"<-"),h.a.createElement("button",{onClick:this.onNextFrame},"->"))),h.a.createElement(u.a,{default:{x:l,y:c,width:i,height:a},lockAspectRatio:!0,onResize:this.onResize},h.a.createElement(d.Stage,{width:i,height:a},h.a.createElement(d.Layer,null,h.a.createElement(d.Image,{image:s,width:i,height:a}),p)),h.a.createElement("button",{onClick:this.onFixup},"Fix"),h.a.createElement("button",{onClick:this.onAnnotationSave},"Save"),v)}}]),a}(h.a.Component);t.a=g},410:function(e,t,a){e.exports=a(412)},412:function(e,t,a){"use strict";a.r(t);var n=a(0),i=a.n(n),s=a(19),r=a.n(s),o=a(167),l=a.n(o),c=(a(229),a(230),a(71)),h=a(60),u=a(70),d=a(56),p=a(2),m=a(3),g=a(5),f=a(4),v=a(444),b=a(276),y=Object(b.a)({palette:{primary:{main:"#ffffff"},secondary:{main:"#5f5f5f"}}}),k=a(449),E=function(e){Object(g.a)(a,e);var t=Object(f.a)(a);function a(e){var n;return Object(p.a)(this,a),(n=t.call(this,e)).handleClick=function(){n.state.isTimerOn?(n.setState({isTimerOn:!1,isSessionEnd:!0}),n.props.stateManager.socket.emit("terminateAgent",{turk_experiment_id:n.props.stateManager.getTurkExperimentId(),mephisto_agent_id:n.props.stateManager.getMephistoAgentId(),turk_worker_id:n.props.stateManager.getTurkWorkerId()})):(n.setState({isTimerOn:!0,startTime:Date.now()}),n.timer=setInterval((function(){n.setState({timeElapsed:Date.now()-n.state.startTime})}),10))},n.state={isTimerOn:!1,isSessionEnd:!1,startTime:0,timeElapsed:0},n}return Object(m.a)(a,[{key:"render",value:function(){var e=this.state.timeElapsed,t=("0"+Math.floor(e/1e3)%60).slice(-2),a=("0"+Math.floor(e/6e4)%60).slice(-2);return i.a.createElement(k.a,{theme:y},i.a.createElement("div",{className:"App"},i.a.createElement("div",{className:"content"},this.state.isSessionEnd?i.a.createElement("p",{style:{fontSize:40}},"Thanks for interacting with the bot. You may leave the page now."):i.a.createElement("div",null,i.a.createElement("div",{style:{fontSize:40}},a," : ",t),i.a.createElement("br",null),i.a.createElement(v.a,{className:"MsgButton",variant:"contained",color:this.state.isTimerOn?"secondary":"primary",onClick:this.handleClick.bind(this)},this.state.isTimerOn?"End":"Start"),i.a.createElement("br",null),i.a.createElement("p",null,"Please click on the button to start the session. "),i.a.createElement("p",null,"When you finished, click on the button to end the session and proceed to next steps.")))))}}]),a}(n.Component),M=a(9),w=a(44);a(203);window.React=i.a,window.ReactDOM=r.a;var x={settings:{showPopoutIcon:!1},content:[{type:"row",content:[{type:"column",content:[{type:"stack",content:[{title:"Interact",type:"react-component",component:"InteractApp",props:{stateManager:M.a}}]},{type:"stack",content:[{title:"Chat History",type:"react-component",component:"History",props:{stateManager:M.a}}]}]},{type:"column",content:[{type:"stack",content:[{title:"VoxelWorld",type:"react-component",component:"VoxelWorld",props:{stateManager:M.a}}],height:60},{type:"stack",content:[{title:"Info",type:"react-component",component:"TurkInfo",props:{stateManager:M.a}},{title:"Settings",type:"react-component",component:"Settings",props:{stateManager:M.a}}]}]}]}]},j=new l.a(x);j.registerComponent("MainPane",c.a),j.registerComponent("Settings",h.a),j.registerComponent("TurkInfo",E),j.registerComponent("History",u.a),j.registerComponent("VoxelWorld",d.a),j.registerComponent("InteractApp",w.a),j.init(),M.a.dashboardLayout=j},42:function(e,t,a){"use strict";var n=a(2),i=a(3),s=a(6),r=a(5),o=a(4),l=a(0),c=a.n(l),h=a(45),u=a(10),d=a(140),p=[["left_ear","left_eye"],["right_ear","right_eye"],["left_eye","right_eye"],["left_eye","nose"],["right_eye","nose"],["right_shoulder","right_elbow"],["right_elbow","right_wrist"],["left_shoulder","left_elbow"],["left_elbow","left_wrist"],["left_shoulder","left_hip"],["left_hip","left_knee"],["left_knee","left_ankle"],["right_shoulder","right_hip"],["right_hip","right_knee"],["right_knee","right_ankle"],["right_hip","left_hip"],["right_shoulder","left_shoulder"]],m=function(e){Object(r.a)(a,e);var t=Object(o.a)(a);function a(e){var i;return Object(n.a)(this,a),(i=t.call(this,e)).onResize=i.onResize.bind(Object(s.a)(i)),i.initialState={height:e.height,width:e.width,rgb:null,humans:null},i.state=i.initialState,i}return Object(i.a)(a,[{key:"onResize",value:function(e,t,a,n,i){this.setState({width:parseInt(a.style.width,10),height:parseInt(a.style.height,10)})}},{key:"componentDidMount",value:function(){this.props.stateManager&&this.props.stateManager.connect(this)}},{key:"componentWillUnmount",value:function(){this.props.stateManager&&this.props.stateManager.disconnect(this)}},{key:"render",value:function(){var e=this.state,t=e.height,a=e.width,n=e.rgb,i=e.humans,s=this.props,r=s.offsetW,o=s.offsetH,l=s.isMobile;if(null===n)return l?c.a.createElement("p",null,"Loading... "):c.a.createElement(h.a,{default:{x:r,y:o,width:a,height:t},lockAspectRatio:!0,onResize:this.onResize},c.a.createElement("p",null,"Loading..."));var m=[],g=0,f=0;return i.forEach((function(e){var a=t/512,n=d.a[g++];g===d.a.length&&(g=0),p.forEach((function(t){var i=e.keypoints[t[0]],s=e.keypoints[t[1]],r=[parseInt(i[0]*a),parseInt(i[1]*a),parseInt(s[0]*a),parseInt(s[1]*a)];m.push(c.a.createElement(u.Line,{key:f++,points:r,stroke:n}))}))})),l?c.a.createElement(u.Stage,{width:a,height:t},c.a.createElement(u.Layer,null,c.a.createElement(u.Image,{image:n,width:a,height:t}),m)):c.a.createElement(h.a,{default:{x:r,y:o,width:a,height:t},lockAspectRatio:!0,onResize:this.onResize},c.a.createElement(u.Stage,{width:a,height:t},c.a.createElement(u.Layer,null,c.a.createElement(u.Image,{image:n,width:a,height:t}),m)))}}]),a}(c.a.Component);t.a=m},43:function(e,t,a){"use strict";var n=a(2),i=a(3),s=a(6),r=a(5),o=a(4),l=a(0),c=a.n(l),h=a(10),u=a(212),d=function(e){Object(r.a)(a,e);var t=Object(o.a)(a);function a(e){var i;return Object(n.a)(this,a),(i=t.call(this,e)).handleDrag=function(e,t){i.setState({memory2d_className:e}),t&&i.setState({drag_coordinates:t})},i.convertGridCoordinate=function(e){var t=i.state,a=t.width,n=t.height,s=t.xmax,r=t.xmin,o=t.ymax,l=t.ymin;return[e[1]*(o-l)/n+l,0,e[0]*(s-r)/a+r]},i.convertCoordinate=function(e){var t=i.state,a=t.width,n=t.height,s=t.xmax,r=t.xmin,o=t.ymax,l=t.ymin,c=parseInt((e[2]-r)/(s-r)*a),h=parseInt((-e[0]-l)/(o-l)*n);return[c,h=n-h]},i.handleWheel=function(e){e.evt.preventDefault();var t=e.target.getStage(),a=t.scaleX(),n=t.getPointerPosition().x/a-t.x()/a,s=t.getPointerPosition().y/a-t.y()/a,r=e.evt.deltaY>0?1.2*a:a/1.2,o=r<1?1:r;i.setState({drag_coordinates:[-(n-t.getPointerPosition().x/o)*o,-(s-t.getPointerPosition().y/o)*o],stageScale:o,stageX:-(n-t.getPointerPosition().x/o)*o,stageY:-(s-t.getPointerPosition().y/o)*o})},i.initialState={height:400,width:600,isLoaded:!1,memory:null,xmin:-10,xmax:10,ymin:-10,ymax:10,bot_xyz:[0,0,0],tooltip:null,stageScale:1,stageX:0,stageY:0,memory2d_className:"memory2d",drag_coordinates:[0,0]},i.state=i.initialState,i.outer_div=c.a.createRef(),i.resizeHandler=i.resizeHandler.bind(Object(s.a)(i)),i}return Object(i.a)(a,[{key:"resizeHandler",value:function(){if(this.props.isMobile){var e=this.props.dimensions;(void 0!==e&&e!==this.state.height||void 0!==e&&e!==this.state.width)&&this.setState({height:e,width:e})}else if(null!=this.outer_div&&null!=this.outer_div.current){var t=this.outer_div.current,a=t.clientHeight,n=t.clientWidth;(void 0!==a&&a!==this.state.height||void 0!==n&&n!==this.state.width)&&this.setState({height:a,width:n})}}},{key:"componentDidMount",value:function(){this.props.stateManager&&this.props.stateManager.connect(this),void 0!==this.props.glContainer&&this.props.glContainer.on("resize",this.resizeHandler)}},{key:"componentDidUpdate",value:function(e,t){var a=this.state.bot_xyz,n=this.state,i=n.xmin,s=n.xmax,r=n.ymin,o=n.ymax,l=a[0],c=a[1],h=!1;l<i&&(i=l,h=!0),l>s&&(s=l,h=!0),c<r&&(r=c,h=!0),c>o&&(o=c,h=!0),h&&this.setState({xmin:i,ymin:r,xmax:s,ymax:o}),this.resizeHandler()}},{key:"componentWillUnmount",value:function(){this.props.stateManager&&this.props.stateManager.disconnect(this)}},{key:"render",value:function(){var e=this;if(!this.state.isLoaded)return c.a.createElement("p",null,"Loading");var t=this.state,a=t.height,n=t.width,i=t.memory,s=t.bot_xyz,r=t.obstacle_map,o=t.tooltip,l=t.drag_coordinates,d=t.stageScale,p=i.objects,m=this.state,g=m.xmin,f=m.xmax,v=m.ymin,b=m.ymax,y=s[1],k=-s[0],E=s[2];if(0===a&&0===n)return c.a.createElement("div",{ref:this.outer_div,style:{height:"100%",width:"100%"}});y=parseInt((y-g)/(f-g)*n),k=parseInt((k-v)/(b-v)*a),k=a-k;var M=[],w=[],x=0;r&&r.forEach((function(e){var t=parseInt((e[0]-g)/(f-g)*n),i=parseInt((e[1]-v)/(b-v)*a);w.push(c.a.createElement(h.Circle,{key:x++,radius:2,x:t,y:i,fill:"#827f7f"}))})),p.forEach((function(t,i,s){var r,o=u.a[Math.abs((r=t.label,r.split("").reduce((function(e,t){return(e=(e<<5)-e+t.charCodeAt(0))&e}),0)))%10],l=t.xyz,d=parseInt((l[2]-g)/(f-g)*n),p=parseInt((-l[0]-v)/(b-v)*a);p=a-p,M.push(c.a.createElement(h.Circle,{key:x++,radius:3,x:d,y:p,fill:o,onMouseEnter:function(a){e.setState({tooltip:JSON.stringify(t,null,4)}),a.currentTarget.setRadius(6)},onMouseLeave:function(t){e.setState({tooltip:null}),t.currentTarget.setRadius(3)}}))})),M.push(c.a.createElement(h.Circle,{key:x++,radius:10,x:y,y:k,fill:"red"})),M.push(c.a.createElement(h.Line,{key:x++,x:y,y:k,points:[0,0,12,0],rotation:180*-E/Math.PI,stroke:"black"}));for(var j=[],S=0;S<n/10;S++)j.push(c.a.createElement(h.Line,{key:12344+S,points:[Math.round(10*S)+.5,0,Math.round(10*S)+.5,a],stroke:"#f5f5f5",strokeWidth:1}));for(j.push(c.a.createElement(h.Line,{key:12344+S++,points:[0,0,10,10]})),x=0;x<a/10;x++)j.push(c.a.createElement(h.Line,{key:12344+S+x,points:[0,Math.round(10*x),n,Math.round(10*x)],stroke:"#ddd",strokeWidth:.5}));for(var O=[],C=this.convertCoordinate([9,0,-9]),I=(C[0]-l[0])/d,_=(C[1]-l[1])/d,R=this.convertGridCoordinate([I,_]),T=.5/d,P=c.a.createElement(h.Line,{key:"axesZ",points:[0,0,n,0],x:I,y:_,stroke:"#AAAAAA",strokeWidth:T}),L=c.a.createElement(h.Line,{key:"axesX",points:[0,-a,0,0],x:I,y:_,stroke:"#AAAAAA",strokeWidth:T}),A=[],D=[C[0]+n,C[1]-a],W=C[0],F=C[1];W<D[0];){W+=30;var H=this.convertGridCoordinate([(W-l[0])/d,0]);A.push(c.a.createElement(h.Text,{key:"textCoordinateX-"+W,text:H[2].toFixed(2),fontSize:10/d,x:(W-10-l[0])/d,y:(C[1]-15-l[1])/d,fill:"#AAAAAA"})),A.push(c.a.createElement(h.Line,{key:"coordinateX-"+W,points:[0,-3/d,0,3/d],x:(W-l[0])/d,y:_,stroke:"#AAAAAA",strokeWidth:T}))}for(;F>D[1];){F-=20;var N=this.convertGridCoordinate([0,(F-l[1])/d]);A.push(c.a.createElement(h.Text,{key:"textCoordinateY-"+F,text:N[0].toFixed(2),fontSize:10/d,x:(C[0]-35-l[0])/d,y:(F-5-l[1])/d,fill:"#AAAAAA"})),A.push(c.a.createElement(h.Line,{key:"coordinateY-"+F,points:[-3/d,0,3/d,0],x:I,y:(F-l[1])/d,stroke:"#AAAAAA",strokeWidth:T}))}var z=[(C[0]-25-l[0])/d,(C[1]+10-l[1])/d];return A.push(c.a.createElement(h.Text,{key:"root-text",fill:"#AAAAAA",text:"".concat(R[0].toFixed(2),", ").concat(R[2].toFixed(2)),fontSize:10/d,x:z[0],y:z[1]})),A.push(c.a.createElement(h.Text,{key:"x-text",fill:"#AAAAAA",text:"x",fontSize:12/d,x:(this.convertCoordinate([-9,0,-8.8])[0]-l[0])/d,y:(this.convertCoordinate([-9,0,-8.8])[1]-l[1])/d})),A.push(c.a.createElement(h.Text,{key:"z-text",fill:"#AAAAAA",text:"z",fontSize:12/d,x:(this.convertCoordinate([9.1,0,9])[0]-l[0])/d,y:(this.convertCoordinate([9.1,0,9])[1]-l[1])/d})),O.push(L,P,A),c.a.createElement("div",{ref:this.outer_div,style:{height:"100%",width:"100%"}},c.a.createElement(h.Stage,{draggable:!0,className:this.state.memory2d_className,width:n,height:a,onWheel:this.handleWheel,scaleX:this.state.stageScale,scaleY:this.state.stageScale,x:this.state.stageX,y:this.state.stageY,onDragMove:function(t){return e.handleDrag("memory2d dragging-memory2d",[t.target.attrs.x,t.target.attrs.y])},onDragEnd:function(t){return e.handleDrag("memory2d",[t.target.attrs.x,t.target.attrs.y])}},c.a.createElement(h.Layer,{className:"gridLayer"},j),c.a.createElement(h.Layer,{className:"coordinateAxesLayer"},O),c.a.createElement(h.Layer,{className:"mapBoundary"},w),c.a.createElement(h.Layer,{className:"renderedObjects"},M),c.a.createElement(h.Layer,null,c.a.createElement(h.Text,{text:o,offsetX:-12,offsetY:-12,visible:null!=o,shadowEnabled:!0}))))}}]),a}(c.a.Component);t.a=d},44:function(e,t,a){"use strict";var n=a(17),i=a(2),s=a(3),r=a(6),o=a(5),l=a(4),c=a(0),h=a.n(c),u=(a(143),a(444)),d=a(211),p=a(182),m=a(443),g=a(210),f=a(132),v=a.n(f),b=(a(144),new window.webkitSpeechRecognition);b.lang="en-US";var y=function(e){Object(o.a)(a,e);var t=Object(l.a)(a);function a(e){var n;return Object(i.a)(this,a),(n=t.call(this,e)).state={recognizing:!1},n.toggleListen=n.toggleListen.bind(Object(r.a)(n)),n.listen=n.listen.bind(Object(r.a)(n)),n.elementRef=h.a.createRef(),n.bindKeyPress=n.handleKeyPress.bind(Object(r.a)(n)),n}return Object(s.a)(a,[{key:"renderChatHistory",value:function(e){var t=this;return this.props.chats.map((function(a,n){return h.a.cloneElement(h.a.createElement(p.a,null,h.a.createElement(g.a,null,a.msg+(a.msg?"Sent successfully"===e?" \u2705":" \u274c":"")),h.a.createElement(m.a,null,""!==a.msg?h.a.createElement(u.a,{style:{backgroundColor:"red",color:"white"},onClick:function(){return t.props.goToQuestion(n)}},"Mark Error"):null)),{key:n.toString()})}))}},{key:"isMounted",value:function(){return null!=this.elementRef.current}},{key:"handleKeyPress",value:function(e){"Enter"===e.key&&(e.preventDefault(),this.handleSubmit())}},{key:"componentDidMount",value:function(){document.addEventListener("keypress",this.bindKeyPress)}},{key:"componentWillUnmount",value:function(){document.removeEventListener("keypress",this.bindKeyPress)}},{key:"toggleListen",value:function(){console.log("togglelisten"),this.setState({recognizing:!this.state.recognizing},this.listen)}},{key:"listen",value:function(){this.state.recognizing?b.start():b.stop(),b.onresult=function(e){for(var t="",a=0;a<e.results.length;++a)e.results[a].isFinal&&(t+=e.results[a][0].transcript);document.getElementById("msg").innerHTML=t},b.onerror=function(e){"not-allowed"===e.error&&(alert("Please grant access to microphone"),b.stop())}}},{key:"handleSubmit",value:function(){var e=document.getElementById("msg").innerHTML;""!==e.replace(/\s/g,"")&&(this.props.setInteractState({msg:e,failed:!1}),this.props.stateManager.logInteractiondata("text command",e),this.props.stateManager.socket.emit("sendCommandToAgent",e),document.getElementById("msg").innerHTML="")}},{key:"render",value:function(){return h.a.createElement("div",{className:"Chat"},h.a.createElement("p",null,"Enter the command to the bot in the input box below, or click the mic button to start/stop voice input."),h.a.createElement("p",null,"Click the x next to the message if the outcome wasn't as expected."),h.a.createElement(v.a,{className:"ASRButton",variant:"contained",color:this.state.recognizing?"default":"secondary",fontSize:"large",onClick:this.toggleListen.bind(this)}),h.a.createElement("p",null," ",this.state.recognizing?"Listening...":""," "),h.a.createElement(d.a,null,this.renderChatHistory(this.props.status)),!0===this.props.isMobile?h.a.createElement("div",{style:{outline:" solid 1px black"},contentEditable:"true",className:"Msg single-line",id:"msg",suppressContentEditableWarning:!0}," "):h.a.createElement("div",{contentEditable:"true",className:"Msg single-line",id:"msg",suppressContentEditableWarning:!0}," "),h.a.createElement(u.a,{className:"MsgButton",variant:"contained",color:"primary",onClick:this.handleSubmit.bind(this)}," ","Submit"," "),h.a.createElement("p",{id:"assistantReply"},this.props.agent_reply," "))}}]),a}(c.Component),k=a(463),E=(a(146),function(e){Object(o.a)(a,e);var t=Object(l.a)(a);function a(e){var n;return Object(i.a)(this,a),(n=t.call(this,e)).state={view:0,parsing_error:!1,action_dict:{},feedback:""},n}return Object(s.a)(a,[{key:"finishQuestions",value:function(){var e={action_dict:this.state.action_dict,parsing_error:this.state.parsing_error,msg:this.props.chats[this.props.failidx].msg,feedback:this.state.feedback};this.props.stateManager.socket.emit("saveErrorDetailsToCSV",e),this.props.goToMessage()}},{key:"saveFeedback",value:function(e){this.setState({feedback:e.target.value})}},{key:"renderParsingFail",value:function(){return h.a.createElement("div",null,h.a.createElement("h3",null," ","Thanks for letting me know that I didn't understand the command right."," "),h.a.createElement(u.a,{variant:"contained",color:"primary",onClick:this.finishQuestions.bind(this)},"Done"))}},{key:"renderParsingSuccess",value:function(){var e=this;return h.a.createElement("div",null,h.a.createElement("h3",null," ","Okay, looks like I understood your command but couldn't follow it all the way through. Tell me more about what I did wrong :"," "),h.a.createElement(k.a,{style:{backgroundColor:"white"},id:"outlined-uncontrolled",label:"",margin:"normal",variant:"outlined",onChange:function(t){return e.saveFeedback(t)}}),h.a.createElement("div",null),h.a.createElement(u.a,{variant:"contained",color:"primary",onClick:this.finishQuestions.bind(this)},"Done"))}},{key:"renderEnd",value:function(){var e=this;return h.a.createElement("div",null,h.a.createElement("h3",null," Thanks! Submit any other feedback here: "),h.a.createElement(k.a,{style:{backgroundColor:"white"},id:"outlined-uncontrolled",label:"",margin:"normal",variant:"outlined",onChange:function(t){return e.saveFeedback(t)}}),h.a.createElement("div",null),h.a.createElement(u.a,{variant:"contained",color:"primary",onClick:this.finishQuestions.bind(this)},"Done"))}},{key:"renderSemanticParserErrorQuestion",value:function(){var e=this;if(this.state.action_dict)if("dialogue_type"in this.state.action_dict){var t=this.state.action_dict.dialogue_type,a="";if("HUMAN_GIVE_COMMAND"===t){var n=this.state.action_dict.action_sequence[0],i=n.action_type.toLowerCase();a="to "+i+" ",["build","dig"].indexOf(i)>=0?("schematic"in n&&(a=a+"'"+n.schematic.text_span+"'"),"location"in n&&(a=a+" at location '"+n.location.text_span+"'"),a+=" ?"):["destroy","fill","spawn","copy","get","scout"].indexOf(i)>=0?("reference_object"in n&&(a=a+"'"+n.reference_object.text_span+"'"),"location"in n&&(a=a+" at location '"+n.location.text_span+"'"),a+=" ?"):["move"].indexOf(i)>=0?("location"in n&&(a=a+" at location '"+n.location.text_span+"'"),a+=" ?"):["stop","resume","undo"].indexOf(i)>=0&&("target_action_type"in n&&(a=a+" at location '"+n.target_action_type+"'"),a+=" ?")}else"GET_MEMORY"===t?a="to answer a question about something in the Minecraft world ?":"PUT_MEMORY"===t?a="to remember or learn something you taught it ?":"NOOP"===t&&(a="to do nothing ?")}else a="did you want me to do nothing ?";return h.a.createElement("div",{className:"question"},h.a.createElement("h5",null,"Did you want the assistant ",a),h.a.createElement(d.a,{className:"answers",component:"nav"},h.a.createElement(p.a,{button:!0,onClick:function(){return e.answerParsing(1)}},h.a.createElement(g.a,{primary:"Yes"})),h.a.createElement(p.a,{button:!0,onClick:function(){return e.answerParsing(2)}},h.a.createElement(g.a,{primary:"No"}))))}},{key:"answerParsing",value:function(e){1===e?this.setState({view:2}):2===e&&this.setState({parsing_error:!0,view:1})}},{key:"goToEnd",value:function(e){this.setState({view:3,new_action_dict:e})}},{key:"componentDidMount",value:function(){var e=this.props.stateManager.memory.chatResponse,t=this.props.chats[this.props.failidx].msg;this.setState({action_dict:e[t]})}},{key:"render",value:function(){return h.a.createElement("div",null,h.a.createElement("div",{className:"msg-header"},"Message you sent to the bot: ",h.a.createElement("br",null),this.props.chats[this.props.failidx].msg),0===this.state.view?this.renderSemanticParserErrorQuestion():null,1===this.state.view?this.renderParsingFail():null,2===this.state.view?this.renderParsingSuccess():null,3===this.state.view?this.renderEnd():null)}}]),a}(c.Component)),M=function(e){Object(o.a)(a,e);var t=Object(l.a)(a);function a(e){var n;return Object(i.a)(this,a),(n=t.call(this,e)).initialState={currentView:1,chatResponse:"",status:"",chats:[{msg:"",failed:!1}],failidx:-1,agent_reply:""},n.state=n.initialState,n.setAssistantReply=n.setAssistantReply.bind(Object(r.a)(n)),n.MessageRef=h.a.createRef(),n}return Object(s.a)(a,[{key:"setInteractState",value:function(e){var t=Object(n.a)(this.state.chats);t.shift(),t.push(e),this.setState({chats:t})}},{key:"setAssistantReply",value:function(e){this.setState({agent_reply:e.agent_reply})}},{key:"componentDidMount",value:function(){this.props.stateManager&&(this.props.stateManager.connect(this),this.props.stateManager.socket.on("showAssistantReply",this.setAssistantReply))}},{key:"componentWillUnmount",value:function(){this.props.stateManager&&this.props.stateManager.disconnect(this)}},{key:"getUrlParameterByName",value:function(e){var t=RegExp("[?&]"+e+"=([^&]*)").exec(window.location.search);return t&&decodeURIComponent(t[1].replace(/\+/g," "))}},{key:"goToMessage",value:function(){var e=this.state.chats;-1!==this.state.failidx&&(e[this.state.failidx].failed=!0),this.setState({currentView:1,chats:e})}},{key:"goToQuestion",value:function(e){this.setState({currentView:2,chats:this.state.chats,failidx:e})}},{key:"render",value:function(){return h.a.createElement("div",{className:"App"},h.a.createElement("div",{className:"content"},1===this.state.currentView?h.a.createElement(y,{status:this.state.status,stateManager:this.props.stateManager,isMobile:this.props.isMobile,ref:this.MessageRef,chats:this.state.chats,agent_reply:this.state.agent_reply,goToQuestion:this.goToQuestion.bind(this),setInteractState:this.setInteractState.bind(this)}):null,2===this.state.currentView?h.a.createElement(E,{stateManager:this.props.stateManager,chats:this.state.chats,failidx:this.state.failidx,goToMessage:this.goToMessage.bind(this),failmsg:this.state.chats[this.state.failidx].msg}):null))}}]),a}(c.Component);t.a=M},46:function(e,t,a){"use strict";function n(e){return(e=e.replace(/_/g," ")).replace(/\w\S*/g,(function(e){return e.charAt(0).toUpperCase()+e.substr(1).toLowerCase()}))}function i(e,t){var a=JSON.parse(t),i=function(e){var t=[];for(var a in e)"logical_form"===a?t.push({event:n(a),description:JSON.stringify(e[a])}):t.push({event:n(a),description:e[a]});return t}(a);e.memory.timelineDetails=i,e.updateTimelineResults();var s={title:n(a.name),cssClass:"scrollable",type:"react-component",component:"TimelineDetails",props:{stateManager:e}};e.dashboardLayout.root.getItemsById("timelineDetails")[0].addChild(s)}a.d(t,"a",(function(){return n})),a.d(t,"b",(function(){return i}))},48:function(e,t,a){"use strict";var n=a(2),i=a(3),s=a(5),r=a(4),o=a(0),l=a.n(o),c=a(17),h=a(6),u=(a(148),a(149),function(e){Object(s.a)(a,e);var t=Object(r.a)(a);function a(e){var i;return Object(n.a)(this,a),(i=t.call(this,e)).state={selectedTags:e.tags||[],suggestions:[],currentValue:""},i.inputRef=l.a.createRef(),i}return Object(i.a)(a,[{key:"render",value:function(){var e=this;return l.a.createElement("div",{className:"tag-selector"},l.a.createElement("div",{className:"tag-holder"},this.state.selectedTags.map((function(t,a){return l.a.createElement("span",{className:"tag-selected",key:a},t,l.a.createElement("button",{onClick:function(){e.removeTag(t)}},"X"))}))),l.a.createElement("input",{type:"text",placeholder:"Tags (press Enter to add)",ref:this.inputRef,onChange:this.update.bind(this),onKeyDown:this.keyDown.bind(this)}),l.a.createElement("div",{className:"tag-suggestions"},this.state.suggestions.map((function(t,a){return l.a.createElement("span",{kkey:a},l.a.createElement("button",{onClick:function(){e.addTag(t)}},a))}))))}},{key:"update",value:function(e){var t=this,a=this.inputRef.current.value;0!==a.length?this.setState({suggestions:this.props.tags.filter((function(e){return e.toLowerCase().slice(0,a.length)===a&&-1===t.state.selectedTags.indexOf(e)}))}):this.setState({suggestions:[]})}},{key:"removeTag",value:function(e){var t=this;this.setState({selectedTags:this.state.selectedTags.filter((function(t){return t!==e}))},(function(){t.props.update(t.state.selectedTags)}))}},{key:"addTag",value:function(e){var t=this;e=e.toLowerCase(),this.setState({selectedTags:this.state.selectedTags.concat(e),suggestions:[]},(function(){t.props.update(t.state.selectedTags)})),this.inputRef.current.value=""}},{key:"keyDown",value:function(e){"Enter"===e.key&&""!==this.inputRef.current.value.trim()&&this.addTag(this.inputRef.current.value),this.setState({currentValue:this.inputRef.current.value})}}]),a}(l.a.Component)),d=function(e){Object(s.a)(a,e);var t=Object(r.a)(a);function a(e){var i;return Object(n.a)(this,a),(i=t.call(this,e)).tags=i.props.tags||[],i.submit=i.submit.bind(Object(h.a)(i)),i.nameRef=l.a.createRef(),i}return Object(i.a)(a,[{key:"render",value:function(){var e=this;return this.props.isMobile?l.a.createElement("div",{className:"data-entry-root",style:{top:this.props.y+"px",left:this.props.x+"px"}},l.a.createElement("input",{placeholder:"Object Name",ref:this.nameRef,defaultValue:this.props.label||""}),l.a.createElement(u,{tags:this.tags,update:function(t){return e.tags=t}}),this.props.includeSubmitButton?l.a.createElement("button",{className:"data-entry-submit",onClick:this.submit},"Submit"):null,this.props.deleteCallback?l.a.createElement("button",{className:"data-entry-delete",onClick:this.props.deleteCallback},"Delete object (\u232b)"):null):l.a.createElement("div",{className:"data-entry-root",style:{top:this.props.y+"px",left:this.props.x+"px",position:"fixed"}},l.a.createElement("input",{placeholder:"Object Name",ref:this.nameRef,defaultValue:this.props.label||""}),l.a.createElement(u,{tags:this.tags,update:function(t){return e.tags=t}}),this.props.includeSubmitButton?l.a.createElement("button",{className:"data-entry-submit",onClick:this.submit},"Submit"):null,this.props.deleteCallback?l.a.createElement("button",{className:"data-entry-delete",onClick:this.props.deleteCallback},"Delete object (\u232b)"):null)}},{key:"submit",value:function(){var e=this.nameRef.current.value.trim();0!==e.length?this.tags.length<2?alert("Please enter at least 2 descriptive tags for the object. Examples include: color, shape, size, orientation, etc."):this.props.onSubmit({name:e.toLowerCase(),tags:this.tags}):alert("You must enter the object's name")}}]),a}(l.a.Component),p=function(e){Object(s.a)(a,e);var t=Object(r.a)(a);function a(e){var i;return Object(n.a)(this,a),(i=t.call(this,e)).state={message:""},i.update=i.update.bind(Object(h.a)(i)),i.drawPointsAndLines=i.drawPointsAndLines.bind(Object(h.a)(i)),i.onClick=i.onClick.bind(Object(h.a)(i)),i.addHandler=i.addHandler.bind(Object(h.a)(i)),i.addMaskHandler=i.addMaskHandler.bind(Object(h.a)(i)),i.addPointHandler=i.addPointHandler.bind(Object(h.a)(i)),i.deleteHandler=i.deleteHandler.bind(Object(h.a)(i)),i.deleteMaskHandler=i.deleteMaskHandler.bind(Object(h.a)(i)),i.deletePointHandler=i.deletePointHandler.bind(Object(h.a)(i)),i.changeTextHandler=i.changeTextHandler.bind(Object(h.a)(i)),i.zoomIn=i.zoomIn.bind(Object(h.a)(i)),i.zoomOut=i.zoomOut.bind(Object(h.a)(i)),i.onMouseMove=i.onMouseMove.bind(Object(h.a)(i)),i.keyDown=i.keyDown.bind(Object(h.a)(i)),i.drawPoint=i.drawPoint.bind(Object(h.a)(i)),i.drawLine=i.drawLine.bind(Object(h.a)(i)),i.localToImage=i.localToImage.bind(Object(h.a)(i)),i.imageToLocal=i.imageToLocal.bind(Object(h.a)(i)),i.shiftViewBy=i.shiftViewBy.bind(Object(h.a)(i)),i.mode=i.props.mode||"default",i.prevMode="default",i.baseMode="default",i.message="",i.currentMaskId=0,i.isDrawingPolygon=!1,i.lastMouse={x:0,y:0},i.points=[[]],i.regions=[],i.newMask="drawing"===i.props.mode,i.canvasRef=l.a.createRef(),i.dataEntryRef=l.a.createRef(),i.zoomPixels=300,i.zoomed=!1,i.pointSize=10,i.color=i.props.color,i}return Object(i.a)(a,[{key:"componentDidMount",value:function(){var e=this;this.canvas=this.canvasRef.current,this.ctx=this.canvas.getContext("2d"),this.points=this.props.masks?this.props.masks.map((function(t){return t.map((function(t){return{x:t.x*e.canvas.width,y:t.y*e.canvas.height}}))})).filter((function(e){return e})):[[]],this.img=this.props.img,this.Offset={x:0,y:0},this.baseScale=Math.min(this.canvas.width/this.img.width,this.canvas.height/this.img.height),this.scale=this.baseScale,this.update()}},{key:"render",value:function(){var e=this,t="500px";this.props.imageWidth&&(t=this.props.imageWidth);var a=0,n=0;return this.props.isMobile?(a=this.canvas&&this.canvas.getBoundingClientRect().left,n=this.canvas&&this.canvas.getBoundingClientRect().bottom+50):(a=this.canvas&&this.canvas.getBoundingClientRect().right,n=this.canvas&&this.canvas.getBoundingClientRect().top+this.canvas.height/3),l.a.createElement("div",null,l.a.createElement("p",null,this.state.message),l.a.createElement("div",null,l.a.createElement("button",{onClick:this.addHandler},l.a.createElement("span",{role:"img","aria-label":"Plus"},"\u2795")," ","(a)"),l.a.createElement("button",{onClick:this.deleteHandler},l.a.createElement("span",{role:"img","aria-label":"Trash"},"\ud83d\uddd1\ufe0f")," ","(d)"),l.a.createElement("button",{onClick:this.zoomIn},l.a.createElement("span",{role:"img","aria-label":"Magnifying Glass Right"},"\ud83d\udd0e")," ","(=)"),l.a.createElement("button",{onClick:this.zoomOut},l.a.createElement("span",{role:"img","aria-label":"Magnifying Glass Left"},"\ud83d\udd0d")," ","(-)"),l.a.createElement("button",{onClick:this.dataEntryRef.current&&this.dataEntryRef.current.submit},l.a.createElement("span",{role:"img","aria-label":"Save"},"\ud83d\udcbe")," ","(\u21b5)")),l.a.createElement("div",{style:{display:"flex",flexDirection:"row"}},l.a.createElement("canvas",{ref:this.canvasRef,width:t,height:t,tabIndex:"0",onClick:this.onClick,onMouseMove:this.onMouseMove,onKeyDown:this.keyDown}),this.props.isMobile&&l.a.createElement("button",{onClick:this.pressEnterOnMobile.bind(this)},"Finished with ",this.props.object,"'s label"),l.a.createElement("div",null,l.a.createElement(d,{ref:this.dataEntryRef,x:a,y:n,onSubmit:this.changeTextHandler,isMobile:this.props.isMobile,label:this.props.object,tags:this.props.tags,deleteCallback:function(){return e.props.deleteLabelHandler()}}))))}},{key:"update",value:function(){this.resetImage(this.zoomed),this.updateMessage();var e=["dragging","focus"].includes(this.mode);this.drawPointsAndLines(e),this.drawRegions(e),"Enter"===this.lastKey&&this.points[this.currentMaskId]&&this.points[this.currentMaskId].length>=3&&(this.zoomed=!1,this.resetImage(this.zoomed),this.drawPointsAndLines(),this.drawRegions())}},{key:"onClick",value:function(e){if("dragging"===this.mode){var t=this.prevMode;return this.prevMode=this.mode,this.mode=t,void this.update()}var a=this.getPointClick();if(null!=a){if("adding"===this.mode&&this.addPointHandler(),"deleting"===this.mode&&this.deletePointHandler(),["addingMask"].includes(this.mode)&&a[0]!==this.currentMaskId)return;if("addingPoint"===this.mode){var n=this.points[a[0]].slice();n.splice(a[1],0,n[a[1]]),this.points[a[0]]=n}return"deletingPoint"===this.mode&&this.points[a[0]].length>3?(this.points[a[0]].length>3&&this.points[a[0]].splice(a[1],1),void this.update()):(this.prevMode=this.mode,this.mode="dragging",this.draggingIndex=a,this.currentMaskId=a[0],void this.update())}if("adding"===this.mode&&this.addMaskHandler(),["drawing","addingMask"].includes(this.mode)&&("Enter"!==this.lastKey||["drawing","addingMask"].includes(this.prevMode)||this.points[this.currentMaskId]&&this.points[this.currentMaskId].length<3))return this.points[this.currentMaskId].push(this.localToImage(this.lastMouse)),this.updateZoom(),this.update(),void(this.lastKey="Mouse");var i=this.getRegionClick();if("default"!==this.mode){if("focus"===this.mode)return-1===i&&(this.prevMode=this.mode,this.mode="default",this.currentMaskId=-1),this.update(),void(this.lastKey="Mouse");if(["deletingMask","deleting"].includes(this.mode)){if(-1===i)return;return"deleting"===this.mode&&this.deleteMaskHandler(),1===this.points.length&&this.props.deleteLabelHandler(),this.points.splice(i,1),this.update(),void(this.lastKey="Mouse")}}else-1!==i&&(this.prevMode=this.mode,this.mode="focus",this.currentMaskId=i,this.update(),this.lastKey="Mouse")}},{key:"keyDown",value:function(e){switch(e.key){case" ":this.points[this.currentMaskId].length>0&&this.points[this.currentMaskId].pop(),this.update();break;case"a":this.addHandler();break;case"s":this.addMaskHandler();break;case"q":this.addPointHandler();break;case"d":this.deleteHandler();break;case"f":this.deleteMaskHandler();break;case"e":this.deletePointHandler();break;case"Backspace":this.props.deleteLabelHandler();break;case"Enter":"Enter"===this.lastKey&&this.points[this.currentMaskId]&&this.points[this.currentMaskId].length>=3&&(this.lastKey=null,this.baseMode="default",this.prevMode=this.mode,this.mode="default",this.save()),["adding","addingMask","addingPoint","deleting","deletingMask","deletingPoint"].includes(this.mode)&&this.points[this.currentMaskId]&&this.points[this.currentMaskId].length>=3&&(this.baseMode="default",this.prevMode=this.mode,this.mode="default");break;case"Escape":if(this.points[this.currentMaskId]){if(this.points[this.currentMaskId].length>=3)return this.save(),this.mode="default",void this.props.exitCallback();if(1===this.points.length)return void this.props.deleteLabelHandler();this.points.splice(this.currentMaskId,1),this.currentMaskId=0,this.prevMode=this.mode,this.mode="default"}break;case"=":this.zoomIn(),this.update();break;case"-":this.zoomOut(),this.update()}this.lastKey=e.key}},{key:"addHandler",value:function(){this.baseMode="default",this.prevMode=this.mode,this.mode="adding"}},{key:"addMaskHandler",value:function(){!this.points[this.currentMaskId]||this.points[this.currentMaskId].length<3||(this.currentMaskId=this.points.length,this.points.push([]),this.prevMode=this.mode,this.mode="addingMask")}},{key:"addPointHandler",value:function(){this.baseMode=this.points[this.currentMaskId]&&0===this.points[this.currentMaskId].length?"default":this.mode,this.prevMode=this.mode,this.mode="addingPoint"}},{key:"deleteHandler",value:function(){this.baseMode="default",this.prevMode=this.mode,this.mode="deleting"}},{key:"deleteMaskHandler",value:function(){!this.points[this.currentMaskId]||this.points[this.currentMaskId].length<3||(this.prevMode=this.mode,this.mode="deletingMask")}},{key:"deletePointHandler",value:function(){this.baseMode=["drawing","addingMask"].includes(this.mode)?this.mode:"default",this.prevMode=this.mode,this.mode="deletingPoint"}},{key:"changeTextHandler",value:function(e){!this.points[this.currentMaskId]||this.points[this.currentMaskId].length<3||(this.save(),this.props.dataEntrySubmit(e))}},{key:"zoomIn",value:function(){this.zoomPixels-=10,this.updateZoom()}},{key:"zoomOut",value:function(){this.zoomPixels+=10,this.updateZoom()}},{key:"updateMessage",value:function(){var e="";switch(this.mode){case"adding":e="Click a point to duplicate it (q) or click anywhere else to create a new mask for this object (s)";break;case"addingMask":e="Create a new mask for this object";break;case"addingPoint":e="Click a point to duplicate it and place it where you want";break;case"deleting":e="Click a point to delete it (e) or click a mask to delete it (f)";break;case"deletingMask":e="Select which mask to delete";break;case"deletingPoint":e="Select a point to delete";break;default:e="Please trace the "+(this.props.object||"object")}e!==this.state.message&&this.setState({message:e})}},{key:"updateZoom",value:function(){if(this.zoomed=!0,this.scale=Math.min(this.canvas.width/this.zoomPixels,this.canvas.height/this.zoomPixels),-1!==this.currentMaskId&&this.points[this.currentMaskId]&&0!==this.points[this.currentMaskId].length&&["default","dragging","drawing","addingMask"].includes(this.mode)){var e=this.points[this.currentMaskId];this.Offset={x:-(e[e.length-1].x-this.zoomPixels/2)*this.scale,y:-(e[e.length-1].y-this.zoomPixels/2)*this.scale}}}},{key:"onMouseMove",value:function(e){var t=this.canvas.getBoundingClientRect();this.lastMouse={x:e.clientX-t.left,y:e.clientY-t.top+1},"dragging"===this.mode&&(this.points[this.draggingIndex[0]][this.draggingIndex[1]]=this.localToImage(this.lastMouse)),this.update()}},{key:"pressEnterOnMobile",value:function(){this.lastKey=null,this.props.submitCallback(this.points)}},{key:"save",value:function(){var e=this;this.props.submitCallback(this.points.map((function(t){return t.map((function(t){return{x:t.x/e.canvas.width,y:t.y/e.canvas.height}}))})),this.newMask)}},{key:"getPointClick",value:function(){for(var e=0;e<this.points.length;e++)for(var t=0;t<this.points[e].length;t++)if(this.distance(this.points[e][t],this.localToImage(this.lastMouse))<this.pointSize/2)return[e,t];return null}},{key:"getRegionClick",value:function(){for(var e=-1,t=0;t<this.regions.length;t++)this.regions[t]&&this.ctx.isPointInPath(this.regions[t],this.lastMouse.x,this.lastMouse.y)&&(e=t);return e}},{key:"drawPointsAndLines",value:function(){for(var e=arguments.length>0&&void 0!==arguments[0]&&arguments[0],t=0;t<this.points.length;t++)if((!0!==e||t===this.currentMaskId)&&this.points[t]&&0!==this.points[t].length){for(var a=0;a<this.points[t].length-1;a++)this.drawLine(this.points[t][a],this.points[t][a+1]),this.drawPoint(this.points[t][a]);this.drawPoint(this.points[t][this.points[t].length-1]),["drawing","addingMask"].includes(this.mode)||["drawing","addingMask"].includes(this.prevMode)||this.drawLine(this.points[t][0],this.points[t][this.points[t].length-1])}this.points[this.currentMaskId]&&this.points[this.currentMaskId].length>0&&["drawing","addingMask"].includes(this.mode)&&this.drawLine(this.points[this.currentMaskId][this.points[this.currentMaskId].length-1],this.localToImage(this.lastMouse))}},{key:"drawRegions",value:function(){var e=arguments.length>0&&void 0!==arguments[0]&&arguments[0];this.regions=[];for(var t=0;t<this.points.length;t++)if(!e||t===this.currentMaskId){var a=this.drawRegion(this.points[t]);this.regions.push(a)}}},{key:"drawPoint",value:function(e){this.ctx.fillStyle="detector"===this.props.originType?"white":"black",this.distance(e,this.localToImage(this.lastMouse))<this.pointSize/2&&(this.ctx.fillStyle="green"),this.ctx.fillRect(e.x-2.5,e.y-2.5,5,5)}},{key:"drawLine",value:function(e,t){this.ctx.beginPath(),this.ctx.moveTo(e.x,e.y),this.ctx.lineTo(t.x,t.y),this.ctx.strokeStyle="detector"===this.props.originType?"white":"black",this.ctx.stroke()}},{key:"drawRegion",value:function(e){if(e&&!(e.length<3)){var t=new Path2D;t.moveTo(e[0].x,e[0].y);for(var a=1;a<e.length;a++)t.lineTo(e[a].x,e[a].y);return t.closePath(),this.ctx.fillStyle=this.color,this.ctx.fill(t,"evenodd"),t}}},{key:"localToImage",value:function(e){var t,a;return this.zoomed?(t=(e.x-this.Offset.x)/this.scale,a=(e.y-this.Offset.y)/this.scale):(t=e.x/this.baseScale,a=e.y/this.baseScale),{x:Math.min(Math.max(t,0),512),y:Math.min(Math.max(a,0),512)}}},{key:"imageToLocal",value:function(e){var t=this.zoomed?this.scale:this.baseScale,a=this.zoomed?this.Offset.x:0,n=this.zoomed?this.Offset.y:0;return{x:e.x*t+a,y:e.y*t+n}}},{key:"resetView",value:function(){this.Offset={x:0,y:0},this.scale=Math.min(this.canvas.width/this.img.width,this.canvas.height/this.img.height)}},{key:"resetImage",value:function(e){this.ctx.resetTransform(),this.ctx.clearRect(0,0,this.canvas.width,this.canvas.height),e?this.ctx.setTransform(this.scale,0,0,this.scale,this.Offset.x,this.Offset.y):(this.ctx.setTransform(this.baseScale,0,0,this.baseScale,0,0),this.zoomed=!1),this.ctx.drawImage(this.img,0,0)}},{key:"shiftViewBy",value:function(e,t){this.Offset={x:this.Offset.x+e,y:this.Offset.y+t}}},{key:"distance",value:function(e,t){return Math.sqrt(Math.pow(e.x-t.x,2)+Math.pow(e.y-t.y,2))}},{key:"distanceToSegment",value:function(e,t,a){var n=this.distance(t,a);if(0===n)return this.distance(e,t);var i=((e.x-t.x)*(a.x-t.x)+(e.y-t.y)*(a.y-t.y))/n;i=Math.max(0,Math.min(1,i));var s={x:t.x+i*(a.x-t.x),y:t.y+i*(a.y-t.y)};return this.distance(e,s)}}]),a}(l.a.Component),m=function(e){Object(s.a)(a,e);var t=Object(r.a)(a);function a(e){var i;return Object(n.a)(this,a),(i=t.call(this,e)).update=i.update.bind(Object(h.a)(i)),i.onClick=i.onClick.bind(Object(h.a)(i)),i.regions={},i.canvasRef=l.a.createRef(),i.imgRef=l.a.createRef(),i}return Object(i.a)(a,[{key:"componentDidMount",value:function(){this.canvas=this.canvasRef.current,this.ctx=this.canvas.getContext("2d"),this.Offset={x:0,y:0},this.baseScale=this.canvas.width/this.props.img.width,this.scale=this.baseScale,this.canvas.height=this.props.img.height*this.baseScale,this.update()}},{key:"componentDidUpdate",value:function(){this.update()}},{key:"render",value:function(){var e="500px";return this.props.imageWidth&&(e=this.props.imageWidth),this.props.isFromCamera&&this.props.displayImage?l.a.createElement("div",null,l.a.createElement("img",{src:this.props.img.src}),l.a.createElement("canvas",{ref:this.canvasRef,width:e,height:e,tabIndex:"0",onMouseMove:this.onMouseMove,onKeyDown:this.keyDown})):l.a.createElement("div",null,l.a.createElement("canvas",{ref:this.canvasRef,width:e,height:e,tabIndex:"0",onMouseMove:this.onMouseMove,onKeyDown:this.keyDown}))}},{key:"update",value:function(){var e=this;this.ctx.resetTransform(),this.ctx.clearRect(0,0,this.canvas.width,this.canvas.height),this.ctx.setTransform(this.scale,0,0,this.scale,this.Offset.x,this.Offset.y),this.ctx.drawImage(this.props.img,0,0);for(var t=0;t<this.props.objects.length;t++){var a=this.props.objects[t],n=this.props.pointMap[a];if(0!==n.length){this.regions[a]=[];var i=this.props.colors[t%this.props.colors.length]||"rgba(0,200,0,.5)",s="detector"===this.props.originTypeMap[a]?"white":"black";for(var r in n){var o=n[r].map((function(t){return{x:t.x*e.canvas.width,y:t.y*e.canvas.height}})),l=this.drawRegionsAndLines(o,i,s);this.regions[a].push(l)}}}this.canvas.addEventListener("mousedown",this.onClick)}},{key:"onClick",value:function(e){var t=-1;for(var a in this.regions)for(var n in this.regions[a])this.regions[a][n]&&this.ctx.isPointInPath(this.regions[a][n],e.offsetX,e.offsetY)&&(t=a);var i=-1!==t;this.props.onClick(e.clientX,e.clientY,i,t)}},{key:"drawRegionsAndLines",value:function(e,t,a){if(!(e.length<3)){var n=new Path2D;n.moveTo(e[0].x,e[0].y);for(var i=1;i<e.length;i++)n.lineTo(e[i].x,e[i].y),this.drawLine(e[i-1],e[i],a);return this.drawLine(e[0],e[e.length-1],a),n.closePath(),this.ctx.fillStyle=t,this.ctx.fill(n,"evenodd"),n}}},{key:"drawLine",value:function(e,t,a){this.ctx.beginPath(),this.ctx.moveTo(e.x,e.y),this.ctx.lineTo(t.x,t.y),this.ctx.strokeStyle=a,this.ctx.stroke()}},{key:"getCanvasBoundingBox",value:function(){return this.canvas.getBoundingClientRect()}}]),a}(l.a.Component),g=["rgba(0,200,0,.5)","rgba(200,0,0,.5)","rgba(0,100,255,.5)","rgba(255,150,0,.5)","rgba(100,255,200,.5)","rgba(200,200,0,.5)","rgba(0,200,150,.5)","rgba(200,0,200,.5)","rgba(0,204,255,.5)"],f=function(e){Object(s.a)(a,e);var t=Object(r.a)(a);function a(e){var i;Object(n.a)(this,a);var s=(i=t.call(this,e)).props.objects;return i.props.objects||(s=i.props.stateManager.curFeedState.objects),null===s&&(s=[]),i.props.isFromCamera&&(s=[]),i.state={objectIds:Object(c.a)(Array(s.length).keys()),currentMode:"select",currentOverlay:null,currentMaskId:null},i.processProps(s),i.registerClick=i.registerClick.bind(Object(h.a)(i)),i.segRef=l.a.createRef(),i.overtime=!1,i.displayImage=!0,setInterval((function(){i.overtime=!0}),6e4*window.MINUTES),i}return Object(i.a)(a,[{key:"componentDidUpdate",value:function(){var e=this.props.objects;this.props.objects||(e=this.props.stateManager.curFeedState.objects),this.props.isFromCamera&&(e=[]),JSON.stringify(e)!==JSON.stringify(this.objects)&&(this.setState({objectIds:Object(c.a)(Array(e.length).keys()),currentMode:"select",currentOverlay:null,currentMaskId:null}),this.processProps(e))}},{key:"render",value:function(){var e=this;if(["draw_polygon","start_polygon"].includes(this.state.currentMode)){this.displayImage=!1;var t=this.state.objectIds.indexOf(parseInt(this.state.currentMaskId)),a=t>=0?g[t%g.length]:g[this.state.objectIds.length%g.length];return l.a.createElement(p,{img:this.image,object:this.drawing_data.name,tags:this.drawing_data.tags,masks:this.pointMap[this.state.currentMaskId],isMobile:this.props.isMobile,originType:this.originTypeMap[this.state.currentMaskId],color:a,exitCallback:function(){e.setState({currentMode:"select"})},submitCallback:this.drawingFinished.bind(this),deleteLabelHandler:this.deleteLabelHandler.bind(this),dataEntrySubmit:this.dataEntrySubmit.bind(this),mode:"start_polygon"===this.state.currentMode?"drawing":null})}return l.a.createElement("div",null,l.a.createElement("p",null,"Label and outline as ",l.a.createElement("b",null,"many objects as possible.")," Click an object in the image to start. ",this.state.objectIds.length," ","object(s) labeled."),this.state.currentOverlay,l.a.createElement("button",{onClick:function(){return e.newPolygon.bind(e)()}},"New Label"),l.a.createElement("div",null,this.state.objectIds.map((function(t,a){return l.a.createElement("button",{key:t,style:{backgroundColor:e.fullColor(g[a%g.length])},onClick:function(){return e.labelSelectHandler(t)}},e.nameMap[t])}))),l.a.createElement(m,{ref:this.segRef,img:this.image,isFromCamera:this.props.isFromCamera,objects:this.state.objectIds,pointMap:this.pointMap,originTypeMap:this.originTypeMap,colors:g,imageWidth:this.props.imageWidth,onClick:this.registerClick,displayImage:this.displayImage}),l.a.createElement("button",{onClick:this.submit.bind(this)},"Finished annotating objects"))}},{key:"processProps",value:function(e){var t=this;this.nextId=e.length,this.objects=e,this.nameMap={},this.pointMap={},this.propertyMap={},this.originTypeMap={};for(var a=0;a<e.length;a++){var n=e[a];this.nameMap[a]=n.label,this.pointMap[a]=n.mask,this.parsePoints(a),this.propertyMap[a]=n.properties,this.originTypeMap[a]=n.type}void 0!==this.props.image?this.image=this.props.image:(this.image=new Image,this.image.onload=function(){t.forceUpdate()},this.image.src=this.props.imgUrl)}},{key:"parsePoints",value:function(e){for(var t in this.pointMap[e])if(this.pointMap[e][t].length<3)delete this.pointMap[e][t];else{var a=0,n=1,i=0,s=1;for(var r in this.pointMap[e][t]){var o=this.pointMap[e][t][r];a=Math.max(o.x,a),n=Math.min(o.x,n),i=Math.max(o.y,i),s=Math.min(o.y,s)}var l=a-n+i-s,c=l<.06?3:50*l;if(this.pointMap[e][t].length>c){for(var h=[],u=this.pointMap[e][t].length/c,d=0;d<this.pointMap[e][t].length-1;d+=u)h.push(this.pointMap[e][t][parseInt(d)]);this.pointMap[e][t]=h}}}},{key:"drawingFinished",value:function(e,t){if(this.pointMap[this.state.currentMaskId]=e,this.setState({currentMode:"select",objectIds:t?this.state.objectIds.splice(0).concat(this.state.currentMaskId):this.state.objectIds}),t){var a=l.a.createElement(d,{x:this.clickPoint.x,y:this.clickPoint.y,onSubmit:this.dataEntrySubmit.bind(this),includeSubmitButton:!0,isMobile:this.props.isMobile});this.setState({currentMode:"fill_data",currentOverlay:a})}}},{key:"deleteLabelHandler",value:function(){delete this.nameMap[this.state.currentMaskId],delete this.pointMap[this.state.currentMaskId],delete this.propertyMap[this.state.currentMaskId];var e=this.state.objectIds.slice(),t=this.state.objectIds.indexOf(parseInt(this.state.currentMaskId));t>=0&&e.splice(t,1),this.setState({currentMode:"select",currentMaskId:-1,objectIds:e})}},{key:"dataEntrySubmit",value:function(e){this.drawingData=e,this.propertyMap[this.state.currentMaskId]=this.drawingData.tags,this.nameMap[this.state.currentMaskId]=this.drawingData.name,this.setState({currentMode:"select",currentOverlay:null})}},{key:"labelSelectHandler",value:function(e){this.setState({currentMode:"draw_polygon",currentOverlay:null,currentMaskId:e}),this.drawing_data={tags:this.propertyMap[e],name:this.nameMap[e]}}},{key:"registerClick",value:function(e,t,a,n){"select"===this.state.currentMode&&(a?(this.drawing_data={tags:this.propertyMap[n],name:this.nameMap[n]},this.setState({currentMode:"draw_polygon",currentOverlay:null,currentMaskId:n})):"fill_data"!==this.state.currentMode&&this.newPolygon(e,t))}},{key:"newPolygon",value:function(){var e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:-1,t=arguments.length>1&&void 0!==arguments[1]?arguments[1]:-1;if(this.drawing_data={tags:null,name:null},this.setState({currentMode:"start_polygon",currentMaskId:this.nextId}),-1===e){var a=this.segRef.current.getCanvasBoundingBox();e=a.left,t=a.top}this.clickPoint={x:e,y:t},this.nextId+=1}},{key:"fullColor",value:function(e){return e.substring(0,e.length-3)+"1)"}},{key:"submit",value:function(){if(this.state.objectIds.length<window.MIN_OBJECTS)alert("Label more objects, or your HIT may be rejected");else{for(var e in this.propertyMap)"string"===typeof this.propertyMap[e]&&(this.propertyMap[e]=this.propertyMap[e].split("\n "));var t={nameMap:this.nameMap,propertyMap:this.propertyMap,pointMap:this.pointMap};this.props.stateManager.socket.emit("saveObjectAnnotation",t),this.props.stateManager.logInteractiondata("object annotation",t),this.props.stateManager.onObjectAnnotationSave(t),this.props.not_turk}}}]),a}(l.a.Component),v=function(e){Object(s.a)(a,e);var t=Object(r.a)(a);function a(e){var i;return Object(n.a)(this,a),(i=t.call(this,e)).props.image?i.initialState={image:i.props.image}:i.initialState={image:void 0},i.state=i.initialState,i}return Object(i.a)(a,[{key:"componentDidMount",value:function(){this.props.stateManager&&this.props.stateManager.connect(this)}},{key:"render",value:function(){return void 0===this.state.image?null:l.a.createElement(f,{image:this.state.image,objects:this.state.objects,not_turk:!0,stateManager:this.props.stateManager,isMobile:this.props.isMobile,imageWidth:this.props.imageWidth,isFromCamera:this.props.isFromCamera})}}]),a}(l.a.Component);t.a=v},56:function(e,t,a){"use strict";var n=a(2),i=a(3),s=a(5),r=a(4),o=a(0),l=a.n(o),c=function(e){Object(s.a)(a,e);var t=Object(r.a)(a);function a(e){var i;return Object(n.a)(this,a),(i=t.call(this,e)).state={status:"",world_state:{}},i.worldContainerRef=l.a.createRef(),i}return Object(i.a)(a,[{key:"getVoxelWorldInitialState",value:function(){this.props.stateManager.socket.emit("getVoxelWorldInitialState")}},{key:"componentDidUpdate",value:function(){this.worldContainerRef.current.contentWindow.postMessage(this.state,"*")}},{key:"componentDidMount",value:function(){this.props.stateManager&&this.props.stateManager.connect(this),this.getVoxelWorldInitialState()}},{key:"render",value:function(){return l.a.createElement("div",null,l.a.createElement("div",{id:"world-container"},l.a.createElement("iframe",{id:"ifr",src:"VoxelWorld/world.html",title:"Voxel World",width:"900",height:"500",ref:this.worldContainerRef})),l.a.createElement("p",null,"Tip: press 'esc' to leave the voxel world, w/a/s/d to move, space to jump"))}}]),a}(l.a.Component);t.a=c},60:function(e,t,a){"use strict";var n=a(2),i=a(3),s=a(6),r=a(5),o=a(4),l=a(0),c=a.n(l),h=(a(94),a(31)),u=a(213),d=a(214),p=(a(95),a(9)),m={width:600,margin:50},g={minWidth:"60px",display:"inline-block"},f=function(e){Object(r.a)(a,e);var t=Object(o.a)(a);function a(e){var i;return Object(n.a)(this,a),(i=t.call(this,e)).switchAnnotation=function(e){p.a.useDesktopComponentOnMobile=e.target.checked,i.setState({use_old_annotation:p.a.useDesktopComponentOnMobile})},i.initialState={url:i.props.stateManager.url,fps:0,connected:!1,image_quality:-1,image_resolution:-1,use_old_annotation:p.a.useDesktopComponentOnMobile},i.props.isMobile&&(m={margin:50}),i.state=i.initialState,i.handleChange=i.handleChange.bind(Object(s.a)(i)),i.handleSubmit=i.handleSubmit.bind(Object(s.a)(i)),i.setImageSettings=i.setImageSettings.bind(Object(s.a)(i)),i.onImageQualityChange=i.onImageQualityChange.bind(Object(s.a)(i)),i.onImageResolutionChange=i.onImageResolutionChange.bind(Object(s.a)(i)),i}return Object(i.a)(a,[{key:"setImageSettings",value:function(e){this.setState(e)}},{key:"handleChange",value:function(e){this.setState({url:e.target.value})}},{key:"handleSubmit",value:function(e){this.props.stateManager.setUrl(this.state.url),e.preventDefault()}},{key:"handleClearSettings",value:function(e){this.props.stateManager.setDefaultUrl()}},{key:"componentDidMount",value:function(){this.props.stateManager&&(this.props.stateManager.connect(this),this.setState({connected:this.props.stateManager.connected}),this.props.stateManager.socket.on("image_settings",this.setImageSettings))}},{key:"componentWillUnmount",value:function(){this.props.stateManager&&(this.props.stateManager.disconnect(this),this.setState({connected:!1}),this.props.stateManager.socket.off("image_settings",this.setImageSettings))}},{key:"onImageQualityChange",value:function(e){!0===this.state.connected&&this.props.stateManager.socket.emit("update_image_settings",{image_quality:e,image_resolution:this.state.image_resolution})}},{key:"onImageResolutionChange",value:function(e){!0===this.state.connected&&this.props.stateManager.socket.emit("update_image_settings",{image_quality:this.state.image_quality,image_resolution:e})}},{key:"render",value:function(){var e;e=!0===this.state.connected?c.a.createElement("status-indicator",{positive:!0,pulse:!0}):c.a.createElement("status-indicator",{negative:!0,pulse:!0});var t=!1;return-1===this.state.image_quality&&(t=!0),c.a.createElement("div",null,c.a.createElement("form",{onSubmit:this.handleSubmit},c.a.createElement("label",null,"Server / Robot URL:",c.a.createElement("input",{type:"text",value:this.state.url,onChange:this.handleChange})),c.a.createElement("input",{type:"submit",value:"Reconnect"})),c.a.createElement("form",{onSubmit:this.handleClearSettings},c.a.createElement("input",{type:"submit",value:"Clear Saved Settings"})),this.props.isMobile&&c.a.createElement(u.a,{control:c.a.createElement(d.a,{checked:p.a.useDesktopComponentOnMobile,onChange:this.switchAnnotation}),label:"Use Old Annotation Tool"}),c.a.createElement("p",null,"FPS: ",this.state.fps),c.a.createElement("p",null," Connection Status: ",e," "),c.a.createElement("div",{style:m},c.a.createElement("label",{style:g},"Image Quality: \xa0"),c.a.createElement("span",null,this.state.image_quality),c.a.createElement("br",null),c.a.createElement("br",null),c.a.createElement(h.a,{value:this.state.image_quality,min:1,max:100,step:1,onChange:this.onImageQualityChange,disabled:t})),c.a.createElement("div",{style:m},c.a.createElement("label",{style:g},"Image Resolution: \xa0"),c.a.createElement("span",null,this.state.image_resolution),c.a.createElement("br",null),c.a.createElement("br",null),c.a.createElement(h.a,{value:this.state.image_resolution,min:16,max:512,step:16,onChange:this.onImageResolutionChange,disabled:t})))}}]),a}(c.a.Component);t.a=f},70:function(e,t,a){"use strict";var n=a(2),i=a(3),s=a(5),r=a(4),o=a(0),l=a.n(o),c=a(211),h=a(182),u=a(210),d=(a(151),function(e){Object(s.a)(a,e);var t=Object(r.a)(a);function a(){return Object(n.a)(this,a),t.apply(this,arguments)}return Object(i.a)(a,[{key:"renderChatHistory",value:function(){return this.props.stateManager.memory.chats.map((function(e,t){return l.a.cloneElement(l.a.createElement(h.a,{alignItems:"flex-start"},l.a.createElement("div",{className:"chatItem"},""!==e.msg?l.a.createElement(u.a,{primary:e.msg}):null)),{key:t.toString()})}))}},{key:"componentDidMount",value:function(){this.props.stateManager&&this.props.stateManager.connect(this)}},{key:"render",value:function(){return l.a.createElement("div",{className:"history"},l.a.createElement("p",null,"The history of the last 5 commands sent to the bot."),l.a.createElement(c.a,null,this.renderChatHistory()))}}]),a}(l.a.Component));t.a=d},71:function(e,t,a){"use strict";var n=a(2),i=a(3),s=a(5),r=a(4),o=a(0),l=a.n(o),c=a(44),h=a(22),u=a(34),d=a(42),p=function(e){Object(s.a)(a,e);var t=Object(r.a)(a);function a(e){var i;return Object(n.a)(this,a),(i=t.call(this,e)).state={rgb:null,depth:null,objects:null,humans:null},i}return Object(i.a)(a,[{key:"componentDidMount",value:function(){this.props.stateManager&&this.props.stateManager.connect(this)}},{key:"componentWillUnmount",value:function(){this.props.stateManager&&this.props.stateManager.disconnect(this)}},{key:"render",value:function(){var e=this.props.stateManager;return l.a.createElement("div",null,l.a.createElement(h.a,{type:"rgb",height:320,width:320,offsetH:10,offsetW:10,stateManager:e}),l.a.createElement(h.a,{type:"depth",height:320,width:320,offsetH:10,offsetW:340,stateManager:e}),l.a.createElement(u.a,{height:320,width:320,offsetH:340,offsetW:10,stateManager:e}),l.a.createElement(d.a,{height:320,width:320,offsetH:340,offsetW:340,stateManager:e}))}}]),a}(l.a.Component),m=a(56),g=function(e){Object(s.a)(a,e);var t=Object(r.a)(a);function a(e){var i;return Object(n.a)(this,a),(i=t.call(this,e)).state={agentType:"locobot"},i}return Object(i.a)(a,[{key:"componentDidMount",value:function(){this.props.stateManager&&this.props.stateManager.connect(this)}},{key:"render",value:function(){console.log("MainPane rendering agent type: "+this.state.agentType);var e=this.props.stateManager;return"locobot"===this.state.agentType?l.a.createElement("div",null,l.a.createElement(c.a,{stateManager:e}),l.a.createElement(p,{stateManager:e})):"craftassist"===this.state.agentType?l.a.createElement("div",null,l.a.createElement(c.a,{stateManager:e}),l.a.createElement(m.a,{stateManager:e})):(console.log("MainPane received invalid agent type"),l.a.createElement("h1",null,"Error: Invalid agent type"))}}]),a}(l.a.Component);t.a=g},80:function(e,t,a){},83:function(e,t,a){"use strict";var n=a(2),i=a(3),s=a(5),r=a(4),o=a(0),l=a.n(o),c=a(46),h=(a(80),function(e){Object(s.a)(a,e);var t=Object(r.a)(a);function a(){return Object(n.a)(this,a),t.apply(this,arguments)}return Object(i.a)(a,[{key:"componentDidMount",value:function(){this.props.stateManager&&this.props.stateManager.connect(this)}},{key:"jsonToResultsTable",value:function(e){var t=e.name,a=e[{perceive:"chat",interpreter:"tasks_to_push",dialogue:"object",memory:"operation"}[t]],n=e.start_time;return n=n.substring(n.indexOf(" ")),l.a.createElement("table",{className:"fixed"},l.a.createElement("tbody",null,l.a.createElement("tr",null,l.a.createElement("td",null,n),l.a.createElement("td",null,t),l.a.createElement("td",null,a))))}},{key:"render",value:function(){var e=this;return l.a.createElement("div",{className:"subpanel"},l.a.createElement("table",{className:"fixed"},l.a.createElement("thead",null,l.a.createElement("tr",null,l.a.createElement("th",null,"Timestamp"),l.a.createElement("th",null,"Event Type"),l.a.createElement("th",null,"Event Summary")))),l.a.createElement("hr",null),this.props.stateManager.memory.timelineSearchResults&&this.props.stateManager.memory.timelineSearchResults.map((function(t,a){return l.a.createElement("div",{className:"result",key:a,onClick:function(){return Object(c.b)(e.props.stateManager,JSON.stringify(t))}},e.jsonToResultsTable(t),l.a.createElement("hr",null))})))}}]),a}(l.a.Component));t.a=h},84:function(e,t,a){"use strict";var n=a(2),i=a(3),s=a(5),r=a(4),o=a(0),l=a.n(o),c=(a(80),function(e){Object(s.a)(a,e);var t=Object(r.a)(a);function a(){return Object(n.a)(this,a),t.apply(this,arguments)}return Object(i.a)(a,[{key:"shouldComponentUpdate",value:function(){return!1}},{key:"componentDidMount",value:function(){this.props.stateManager&&this.props.stateManager.connect(this)}},{key:"renderTable",value:function(e){if(e)return e.map((function(e,t){var a=e.event,n=e.description;return l.a.createElement("tr",{key:t},l.a.createElement("td",null,l.a.createElement("strong",null,a)),l.a.createElement("td",null,n))}))}},{key:"render",value:function(){return l.a.createElement("div",{className:"subpanel"},l.a.createElement("table",null,l.a.createElement("tbody",null,this.renderTable(this.props.stateManager.memory.timelineDetails))))}}]),a}(l.a.Component));t.a=c},85:function(e,t,a){"use strict";a.d(t,"b",(function(){return T}));var n=a(17),i=a(2),s=a(3),r=a(5),o=a(4),l=a(0),c=a.n(l),h=a(106),u=a(46),d=a(137);function p(e,t){var a=[];if(t){e.memory.timelineSearchPattern=t;var n=new d.a(e.memory.timelineEventHistory,{ignoreLocation:!0,useExtendedSearch:!0}).search("'"+t);n.length&&n.forEach((function(t){var n=t.item,i=JSON.parse(n);e.memory.timelineFilters.includes(Object(u.a)(i.name))&&a.push(i)}))}e.memory.timelineSearchResults=a,e.updateTimelineResults()}var m=a(40),g=a(452),f=a(445),v=a(466),b=a(453),y=a(447),k=a(210),E=a(459),M=a(461),w=Object(g.a)((function(e){return{formControl:{margin:e.spacing(1),minWidth:120,maxWidth:300}}})),x={PaperProps:{style:{maxHeight:224,width:250}}};function j(e){var t=e.stateManager,a=w(),n=T,i=c.a.useState(n),s=Object(m.a)(i,2),r=s[0],o=s[1];return c.a.createElement("div",null,c.a.createElement(y.a,{className:a.formControl},c.a.createElement(v.a,{id:"multiple-checkbox-label"},"Filters"),c.a.createElement(E.a,{labelId:"multiple-checkbox-label",id:"multiple-checkbox",multiple:!0,value:r,onChange:function(e){o(e.target.value),t.memory.timelineFilters=e.target.value,t.updateTimeline(),p(t,t.memory.timelineSearchPattern)},input:c.a.createElement(f.a,null),renderValue:function(e){return e.join(", ")},MenuProps:x},n.map((function(e){return c.a.createElement(b.a,{key:e,value:e},c.a.createElement(M.a,{checked:r.indexOf(e)>-1}),c.a.createElement(k.a,{primary:e}))})))))}var S=a(111),O=a.n(S),C=a(276),I=a(449),_=(a(80),Object(C.a)({palette:{type:"dark"}})),R=new h.a,T=["Perceive","Dialogue","Interpreter","Memory"],P=[{id:"Timeline",content:"Timeline",nestedGroups:T},{id:"Perceive",content:"Perception"},{id:"Dialogue",content:"Dialogue"},{id:"Interpreter",content:"Interpreter"},{id:"Memory",content:"Memory"}],L={tooltip:{followMouse:!0,overflowMethod:"cap",template:function(e,t){var a=JSON.parse(e.title);return"<pre>event: "+a.name+"\nagent time: "+a.agent_time+"</pre>"}},zoomMax:864e5,rollingMode:{follow:!0}},A=function(e){var t=e.onChange,a=e.placeholder;return c.a.createElement("div",{className:"search"},c.a.createElement("input",{className:"searchInput",type:"text",onChange:t,placeholder:a}),c.a.createElement("span",{className:"searchSpan"},c.a.createElement(O.a,null)))},D=function(e){Object(r.a)(a,e);var t=Object(o.a)(a);function a(){var e;return Object(i.a)(this,a),(e=t.call(this)).timeline={},e.appRef=Object(l.createRef)(),e.prevEvent="",e.searchPattern="",e}return Object(s.a)(a,[{key:"componentDidMount",value:function(){this.props.stateManager&&this.props.stateManager.connect(this),this.searchFilters=Object(n.a)(this.props.stateManager.memory.timelineFilters),this.timeline=new h.b(this.appRef.current,R,P,L);var e=this.timeline.getCurrentTime(),t=this.timeline.getCurrentTime();e=e.setSeconds(e.getSeconds()-10),t=t.setSeconds(t.getSeconds()+10),this.timeline.setOptions({start:e,end:t});var a=this;this.timeline.on("click",(function(e){if(e.item){var t=R.get(e.item);Object(u.b)(a.props.stateManager,t.title)}}))}},{key:"renderEvent",value:function(){var e=this.props.stateManager.memory.timelineEvent;if(e&&e!==this.prevEvent){this.prevEvent=e;var t=JSON.parse(e),a="";"perceive"===t.name&&(a=' ("'+t.chat+'")'),R.add([{title:JSON.stringify(t,null,2),content:t.name+a,group:"Timeline",className:t.name,start:t.start_time,end:t.end_time,type:"box"}]),R.add([{title:JSON.stringify(t,null,2),group:Object(u.a)(t.name),className:t.name,start:t.start_time,end:t.end_time,type:"box"}])}}},{key:"toggleVisibility",value:function(){var e=this.props.stateManager.memory.timelineFilters;if(e&&e!==this.searchFilters){console.log(e),this.searchFilters=Object(n.a)(e);for(var t=R.get(),a=0;a<t.length;a++)e.includes(Object(u.a)(t[a].className))?t[a].style="opacity: 1;":t[a].style="opacity: 0.2;";R.update(t)}}},{key:"render",value:function(){var e=this;return this.renderEvent(),this.toggleVisibility(),c.a.createElement("div",{className:"timeline"},c.a.createElement("p",{id:"description"},"A visualizer for viewing, inspecting, and searching through agent activities interactively.",c.a.createElement("br",null),"Click an event to view more details!"),c.a.createElement(A,{placeholder:"Search",onChange:function(t){return p(e.props.stateManager,t.target.value)}}),c.a.createElement("div",{id:"dropdown"},c.a.createElement(I.a,{theme:_},c.a.createElement(j,{stateManager:this.props.stateManager}))),c.a.createElement("div",{className:"clearfloat"}),c.a.createElement("div",{ref:this.appRef}))}}]),a}(c.a.Component);t.a=D},86:function(e,t,a){"use strict";var n=a(40),i=a(2),s=a(3),r=a(5),o=a(4),l=a(0),c=a.n(l),h=a(41),u=a(35);function d(e){return null===e.value||void 0===e.value?null:c.a.createElement(c.a.Fragment,null,c.a.createElement(h.a,{item:!0,xs:4,s:4},c.a.createElement(u.a,{variant:"button",align:"right"},e.title)),c.a.createElement(h.a,{item:!0,xs:8,s:8},c.a.createElement(u.a,{variant:"body1",align:"left"},e.value)))}var p=function(e){Object(r.a)(a,e);var t=Object(o.a)(a);function a(){return Object(i.a)(this,a),t.apply(this,arguments)}return Object(s.a)(a,[{key:"render",value:function(){var e=(0,Object(n.a)(this.props.memoryManager,3)[2])(this.props.uuid);return c.a.createElement(h.a,{container:!0,spacing:2,style:{padding:"16px"}},c.a.createElement(h.a,{item:!0,xs:11},c.a.createElement(u.a,{variant:"h4",align:"left"},"Attributes")),c.a.createElement(d,{title:"UUID",value:e.data[0]}),c.a.createElement(d,{title:"ID",value:e.data[1]}),c.a.createElement(d,{title:"X",value:e.data[2]}),c.a.createElement(d,{title:"Y",value:e.data[3]}),c.a.createElement(d,{title:"Z",value:e.data[4]}),c.a.createElement(d,{title:"Yaw",value:e.data[5]}),c.a.createElement(d,{title:"Pitch",value:e.data[6]}),c.a.createElement(d,{title:"Type",value:e.data[9]}),c.a.createElement(d,{title:"Name",value:e.data[7]}),c.a.createElement(h.a,{item:!0,xs:12},c.a.createElement(u.a,{variant:"h4",align:"left"},"Triples")),c.a.createElement(h.a,{item:!0,xs:4},c.a.createElement(u.a,{variant:"button",align:"left"},"Predicate")),c.a.createElement(h.a,{item:!0,xs:4},c.a.createElement(u.a,{variant:"button",align:"left"},"Value")),c.a.createElement(h.a,{item:!0,xs:4},c.a.createElement(u.a,{variant:"button",align:"left"},"Confidence")),e.triples.map((function(e){return c.a.createElement(c.a.Fragment,null,c.a.createElement(h.a,{item:!0,xs:4},c.a.createElement(u.a,{variant:"body2",align:"left"},e[4])),c.a.createElement(h.a,{item:!0,xs:4},c.a.createElement(u.a,{variant:"body2",align:"left"},e[6])),c.a.createElement(h.a,{item:!0,xs:4},c.a.createElement(u.a,{variant:"body2",align:"left"},e[7])))})))}}]),a}(c.a.Component);t.a=p},87:function(e,t,a){"use strict";var n=a(2),i=a(3),s=a(6),r=a(5),o=a(4),l=a(0),c=a.n(l),h=(a(94),a(31)),u=(a(95),{width:600,margin:50}),d={minWidth:"60px",display:"inline-block"},p=function(e){Object(r.a)(a,e);var t=Object(o.a)(a);function a(e){var i;return Object(n.a)(this,a),(i=t.call(this,e)).initialState={learningRate:.005,trainSplit:.7,maxIters:100,modelMetrics:null},i.onAnnotationSave=i.onAnnotationSave.bind(Object(s.a)(i)),i.onRetrain=i.onRetrain.bind(Object(s.a)(i)),i.onModelSwitch=i.onModelSwitch.bind(Object(s.a)(i)),i.onLearningRateChange=i.onLearningRateChange.bind(Object(s.a)(i)),i.onTrainSplitChange=i.onTrainSplitChange.bind(Object(s.a)(i)),i.onMaxItersChange=i.onMaxItersChange.bind(Object(s.a)(i)),i.state=i.initialState,i}return Object(i.a)(a,[{key:"componentDidMount",value:function(){this.props.stateManager&&this.props.stateManager.connect(this)}},{key:"onAnnotationSave",value:function(){this.props.stateManager&&this.props.stateManager.saveAnnotations()}},{key:"onRetrain",value:function(){if(this.props.stateManager){console.log("retraining detector...");var e=this.state,t=e.learningRate,a=e.trainSplit,n=e.maxIters;this.props.stateManager.socket.emit("retrain_detector",{learningRate:t,trainSplit:a,maxIters:n})}}},{key:"onModelSwitch",value:function(){this.props.stateManager&&(console.log("switching model..."),this.props.stateManager.socket.emit("switch_detector"))}},{key:"onLearningRateChange",value:function(e){this.setState({learningRate:e})}},{key:"onTrainSplitChange",value:function(e){this.setState({trainSplit:e})}},{key:"onMaxItersChange",value:function(e){this.setState({maxIters:e})}},{key:"render",value:function(){var e=null;if(this.state.modelMetrics){var t=this.state.modelMetrics.segm,a=Object.keys(t).map((function(e){return c.a.createElement("div",null,e+": "+t[e])}));e=c.a.createElement("div",null,c.a.createElement("div",null,"New model trained!"),"Evalution: ",a)}return c.a.createElement("div",null,c.a.createElement("div",{style:u},c.a.createElement("label",{style:d},"Learning Rate: \xa0"),c.a.createElement("span",null,this.state.learningRate),c.a.createElement("br",null),c.a.createElement("br",null),c.a.createElement(h.a,{value:this.state.learningRate,min:0,max:1,step:.005,onChange:this.onLearningRateChange})),c.a.createElement("div",{style:u},c.a.createElement("label",{style:d},"Train split: \xa0"),c.a.createElement("span",null,this.state.trainSplit),c.a.createElement("br",null),c.a.createElement("br",null),c.a.createElement(h.a,{value:this.state.trainSplit,min:0,max:1,step:.05,onChange:this.onTrainSplitChange})),c.a.createElement("div",{style:u},c.a.createElement("label",{style:d},"Max iterations: \xa0"),c.a.createElement("span",null,this.state.maxIters),c.a.createElement("br",null),c.a.createElement("br",null),c.a.createElement(h.a,{value:this.state.maxIters,min:50,max:1e3,step:50,onChange:this.onMaxItersChange})),c.a.createElement("button",{onClick:this.onAnnotationSave},"Save Annotations"),c.a.createElement("button",{onClick:this.onRetrain},"Retrain"),c.a.createElement("button",{onClick:this.onModelSwitch},"Switch"),e)}}]),a}(c.a.Component);t.a=p},88:function(e,t,a){"use strict";var n=a(2),i=a(3),s=a(6),r=a(5),o=a(4),l=a(0),c=a.n(l),h=a(9),u=(a(94),a(31)),d=(a(95),{width:600,margin:50}),p={minWidth:"60px",display:"inline-block"},m=function(e){Object(r.a)(a,e);var t=Object(o.a)(a);function a(e){var i;return Object(n.a)(this,a),(i=t.call(this,e)).keyboardToggle=function(){1==i.keyCheckRef.current.checked?i.addKeyListener():i.removeKeyListener(),i.setState({keyboard_enabled:i.keyCheckRef.current.checked})},i.initialState={move:.3,yaw:.01,velocity:.1,data_logging_time:30,keyboard_enabled:!1},i.state=i.initialState,i.handleSubmit=i.handleSubmit.bind(Object(s.a)(i)),i.onYawChange=i.onYawChange.bind(Object(s.a)(i)),i.onDataLoggingTimeChange=i.onDataLoggingTimeChange.bind(Object(s.a)(i)),i.onMoveChange=i.onMoveChange.bind(Object(s.a)(i)),i.onVelocityChange=i.onVelocityChange.bind(Object(s.a)(i)),i.navRef=c.a.createRef(),i.keyCheckRef=c.a.createRef(),i.handleClick=i.handleClick.bind(Object(s.a)(i)),i.logData=i.logData.bind(Object(s.a)(i)),i.stopRobot=i.stopRobot.bind(Object(s.a)(i)),i.unstopRobot=i.unstopRobot.bind(Object(s.a)(i)),i.keyboardToggle=i.keyboardToggle.bind(Object(s.a)(i)),i.addKeyListener=i.addKeyListener.bind(Object(s.a)(i)),i.removeKeyListener=i.removeKeyListener.bind(Object(s.a)(i)),i}return Object(i.a)(a,[{key:"handleSubmit",value:function(e){h.a.setUrl(this.state.url),e.preventDefault()}},{key:"logData",value:function(e){h.a.socket.emit("logData",this.state.data_logging_time),console.log("logData",this.state.data_logging_time)}},{key:"stopRobot",value:function(e){h.a.socket.emit("stopRobot"),console.log("Robot Stopped")}},{key:"unstopRobot",value:function(e){h.a.socket.emit("unstopRobot"),console.log("Robot UnStopped")}},{key:"addKeyListener",value:function(){var e={};this.onkey=function(t){e[t.keyCode]=!0},document.addEventListener("keyup",this.onkey);this.intervalHandle=setInterval(h.a.keyHandler,33.33,e)}},{key:"removeKeyListener",value:function(){document.removeEventListener("keyup",this.onkey),clearInterval(this.intervalHandle)}},{key:"componentDidMount",value:function(){h.a&&h.a.connect(this)}},{key:"onDataLoggingTimeChange",value:function(e){this.setState({data_logging_time:e})}},{key:"onYawChange",value:function(e){this.setState({yaw:e})}},{key:"onMoveChange",value:function(e){this.setState({move:e})}},{key:"onVelocityChange",value:function(e){this.setState({velocity:e})}},{key:"handleClick",value:function(e){var t=e.target.id;"key_up"===t?h.a.keyHandler({38:!0}):"key_down"===t?h.a.keyHandler({40:!0}):"key_left"===t?h.a.keyHandler({37:!0}):"key_right"===t?h.a.keyHandler({39:!0}):"pan_left"===t?h.a.keyHandler({49:!0}):"pan_right"===t?h.a.keyHandler({50:!0}):"tilt_up"===t?h.a.keyHandler({51:!0}):"tilt_down"===t&&h.a.keyHandler({52:!0})}},{key:"render",value:function(){return c.a.createElement("div",{ref:this.navRef},c.a.createElement("br",null),c.a.createElement("br",null),c.a.createElement("div",null,c.a.createElement("label",null," Base "),c.a.createElement("button",{id:"key_up",onClick:this.handleClick},"Up")),c.a.createElement("div",null,c.a.createElement("button",{id:"key_left",onClick:this.handleClick},"Left"),c.a.createElement("button",{id:"key_down",onClick:this.handleClick},"Down"),c.a.createElement("button",{id:"key_right",onClick:this.handleClick},"Right")),c.a.createElement("br",null),c.a.createElement("br",null),c.a.createElement("div",null,c.a.createElement("label",null," Camera "),c.a.createElement("button",{id:"tilt_up",onClick:this.handleClick},"Up")),c.a.createElement("div",null,c.a.createElement("button",{id:"pan_left",onClick:this.handleClick},"Left"),c.a.createElement("button",{id:"tilt_down",onClick:this.handleClick},"Down"),c.a.createElement("button",{id:"pan_right",onClick:this.handleClick},"Right")),c.a.createElement("br",null),c.a.createElement("input",{type:"checkbox",defaultChecked:this.state.keyboard_enabled,ref:this.keyCheckRef,onChange:this.keyboardToggle}),c.a.createElement("label",null," ","Enable Keyboard control (Use the arrow keys to move the robot, keys"," ",2," and ",4," to move camera)"," "),c.a.createElement("div",{style:d},c.a.createElement("label",{style:p},"Rotation (radians): \xa0"),c.a.createElement("span",null,this.state.yaw),c.a.createElement("br",null),c.a.createElement("br",null),c.a.createElement(u.a,{value:this.state.yaw,min:0,max:6.28,step:.01,onChange:this.onYawChange}),c.a.createElement("br",null),c.a.createElement("label",{style:p},"Move Distance (metres): \xa0"),c.a.createElement("span",null,this.state.move),c.a.createElement("br",null),c.a.createElement("br",null),c.a.createElement(u.a,{value:this.state.move,min:0,max:10,step:.1,onChange:this.onMoveChange})),c.a.createElement("div",{style:d},c.a.createElement("label",{style:p},"Velocity: \xa0"),c.a.createElement("span",null,this.state.velocity),c.a.createElement("br",null),c.a.createElement("br",null),c.a.createElement(u.a,{value:this.state.velocity,min:0,max:1,step:.05,onChange:this.onVelocityChange})),c.a.createElement("div",null,c.a.createElement("div",{style:d},c.a.createElement("label",{style:p},"Data Logging Time (seconds): \xa0"),c.a.createElement("span",null,this.state.data_logging_time),c.a.createElement("br",null),c.a.createElement("br",null),c.a.createElement(u.a,{value:this.state.data_logging_time,min:0,max:300,step:1,onChange:this.onDataLoggingTimeChange})),c.a.createElement("button",{id:"log_data",onClick:this.logData},"Log Data")))}}]),a}(c.a.Component);t.a=m},89:function(e,t,a){"use strict";var n=a(2),i=a(3),s=a(5),r=a(4),o=a(0),l=a.n(o),c=a(456),h=a(457),u=a(458),d=(a(152),function(e){Object(s.a)(a,e);var t=Object(r.a)(a);function a(e){var i;return Object(n.a)(this,a),(i=t.call(this,e)).state=e.state,i}return Object(i.a)(a,[{key:"render",value:function(){var e=this;return l.a.createElement(h.a,{style:{overflowX:"scroll",overflowY:"scroll",width:"100%"},className:"Navbar fixed-bottom"},l.a.createElement(u.a,null,l.a.createElement("button",{className:"button",onClick:function(){return e.props.paneHandler("home")}},"Home"),l.a.createElement("button",{className:"button",onClick:function(){return e.props.paneHandler("navigation")}},"Navigation"),l.a.createElement("button",{className:"button",onClick:function(){return e.props.paneHandler("settings")}},"Settings"),l.a.createElement("button",{className:"button",onClick:function(){return e.props.paneHandler("annotation")}},"Annotation"),l.a.createElement("button",{className:"button",onClick:function(){return e.props.paneHandler("camera")}},"Camera")))}}]),a}(l.a.Component)),p=a(454),m=a(455),g=a(22),f=a(42),v=a(9),b=a(44),y=a(43),k=function(e){Object(s.a)(a,e);var t=Object(r.a)(a);function a(){return Object(n.a)(this,a),t.apply(this,arguments)}return Object(i.a)(a,[{key:"render",value:function(){return l.a.createElement("div",null,l.a.createElement(p.a,null,l.a.createElement(m.a,null,"Video Feed 1",l.a.createElement(g.a,{type:"rgb",height:this.props.imageWidth,width:this.props.imageWidth,offsetH:0,offsetW:0,stateManager:v.a,isMobile:!0})),l.a.createElement(m.a,null,"Memory 2d",l.a.createElement(y.a,{stateManager:v.a,isMobile:!0,dimensions:this.props.imageWidth}))),l.a.createElement(p.a,null,l.a.createElement(m.a,null,"Video Feed 2",l.a.createElement(g.a,{type:"depth",height:this.props.imageWidth,width:this.props.imageWidth,offsetH:0,offsetW:0,stateManager:v.a,isMobile:!0})),l.a.createElement(m.a,null,"Video Feed 3",l.a.createElement(f.a,{type:"depth",height:this.props.imageWidth,width:this.props.imageWidth,offsetH:0,offsetW:0,stateManager:v.a,isMobile:!0}))),l.a.createElement(p.a,null,l.a.createElement(b.a,{stateManager:v.a,isMobile:!0})))}}]),a}(l.a.Component),E=(a(153),function(e){Object(s.a)(a,e);var t=Object(r.a)(a);function a(e){var i;return Object(n.a)(this,a),(i=t.call(this,e)).state={commands:[]},i.intervalId=void 0,i}return Object(i.a)(a,[{key:"componentDidMount",value:function(){this.intervalId=setInterval(this.sendAndClearCommands.bind(this),33.33)}},{key:"componentWillUnmount",value:function(){this.intervalId&&clearInterval(this.intervalId)}},{key:"addCommand",value:function(e){var t=this.state.commands;t.push(e),this.setState({commands:t})}},{key:"sendAndClearCommands",value:function(){this.state&&(v.a.buttonHandler(this.state.commands),this.setState({commands:[]}))}},{key:"render",value:function(){var e=this;return l.a.createElement("div",{className:"container"},l.a.createElement("button",{className:"directionButton left",onClick:function(){return e.addCommand("MOVE_LEFT")}},"Turn Left"),l.a.createElement("button",{className:"directionButton up",onClick:function(){return e.addCommand("MOVE_FORWARD")}},"Forward"),l.a.createElement("button",{className:"directionButton down",onClick:function(){return e.addCommand("MOVE_DOWN")}},"Backward"),l.a.createElement("button",{className:"directionButton right",onClick:function(){return e.addCommand("MOVE_RIGHT")}},"Turn Right"))}}]),a}(l.a.Component)),M=function(e){Object(s.a)(a,e);var t=Object(r.a)(a);function a(){return Object(n.a)(this,a),t.apply(this,arguments)}return Object(i.a)(a,[{key:"render",value:function(){return l.a.createElement("div",null,l.a.createElement(p.a,null,l.a.createElement(m.a,null,"Video Feed 1",l.a.createElement(g.a,{type:"rgb",height:this.props.imageWidth,width:this.props.imageWidth,offsetH:0,offsetW:0,stateManager:v.a,isMobile:!0})),l.a.createElement(m.a,null,"Memory 2d",l.a.createElement(y.a,{stateManager:v.a,isMobile:!0,dimensions:this.props.imageWidth}))),l.a.createElement(p.a,null,l.a.createElement(m.a,null,"Video Feed 2",l.a.createElement(g.a,{type:"depth",height:this.props.imageWidth,width:this.props.imageWidth,offsetH:0,offsetW:0,stateManager:v.a,isMobile:!0})),l.a.createElement(m.a,null,"Video Feed 3",l.a.createElement(f.a,{type:"depth",height:this.props.imageWidth,width:this.props.imageWidth,offsetH:0,offsetW:0,stateManager:v.a,isMobile:!0}))),l.a.createElement(E,null))}}]),a}(l.a.Component),w=a(60),x=function(e){Object(s.a)(a,e);var t=Object(r.a)(a);function a(){return Object(n.a)(this,a),t.apply(this,arguments)}return Object(i.a)(a,[{key:"render",value:function(){return l.a.createElement("div",null,l.a.createElement(p.a,null,l.a.createElement(m.a,null,"Video Feed 1",l.a.createElement(g.a,{type:"rgb",height:this.props.imageWidth,width:this.props.imageWidth,offsetH:0,offsetW:0,stateManager:v.a,isMobile:!0})),l.a.createElement(m.a,null,"Memory 2d",l.a.createElement(y.a,{stateManager:v.a,isMobile:!0,dimensions:this.props.imageWidth}))),l.a.createElement(p.a,null,l.a.createElement(m.a,null,"Video Feed 2",l.a.createElement(g.a,{type:"depth",height:this.props.imageWidth,width:this.props.imageWidth,offsetH:0,offsetW:0,stateManager:v.a,isMobile:!0})),l.a.createElement(m.a,null,"Video Feed 3",l.a.createElement(f.a,{type:"depth",height:this.props.imageWidth,width:this.props.imageWidth,offsetH:0,offsetW:0,stateManager:v.a,isMobile:!0}))),l.a.createElement(w.a,{stateManager:v.a,isMobile:!0}))}}]),a}(l.a.Component),j=a(48),S=a(6),O=(a(154),a(155),function(e){Object(s.a)(a,e);var t=Object(r.a)(a);function a(e){var i;return Object(n.a)(this,a),(i=t.call(this,e)).state={selectedTags:e.tags||[],suggestions:[],currentValue:""},i.inputRef=l.a.createRef(),i}return Object(i.a)(a,[{key:"render",value:function(){var e=this;return l.a.createElement("div",{className:"tag-selector"},l.a.createElement("div",{className:"tag-holder"},this.state.selectedTags.map((function(t,a){return l.a.createElement("span",{className:"tag-selected",key:a},t,l.a.createElement("button",{onClick:function(){e.removeTag(t)}},"X"))}))),l.a.createElement("input",{type:"text",placeholder:"Tags (press Enter to add)",ref:this.inputRef,onChange:this.update.bind(this),onKeyDown:this.keyDown.bind(this)}),l.a.createElement("div",{className:"tag-suggestions"},this.state.suggestions.map((function(t,a){return l.a.createElement("span",{kkey:a},l.a.createElement("button",{onClick:function(){e.addTag(t)}},a))}))))}},{key:"update",value:function(e){var t=this,a=this.inputRef.current.value;0!==a.length?this.setState({suggestions:this.props.tags.filter((function(e){return e.toLowerCase().slice(0,a.length)===a&&-1===t.state.selectedTags.indexOf(e)}))}):this.setState({suggestions:[]})}},{key:"removeTag",value:function(e){var t=this;this.setState({selectedTags:this.state.selectedTags.filter((function(t){return t!==e}))},(function(){t.props.update(t.state.selectedTags)}))}},{key:"addTag",value:function(e){var t=this;e=e.toLowerCase(),this.setState({selectedTags:this.state.selectedTags.concat(e),suggestions:[]},(function(){t.props.update(t.state.selectedTags)})),this.inputRef.current.value=""}},{key:"keyDown",value:function(e){"Enter"===e.key&&""!==this.inputRef.current.value.trim()&&this.addTag(this.inputRef.current.value),this.setState({currentValue:this.inputRef.current.value})}}]),a}(l.a.Component)),C=function(e){Object(s.a)(a,e);var t=Object(r.a)(a);function a(e){var i;return Object(n.a)(this,a),(i=t.call(this,e)).tags=i.props.tags||[],i.submit=i.submit.bind(Object(S.a)(i)),i.nameRef=l.a.createRef(),i}return Object(i.a)(a,[{key:"render",value:function(){var e=this;return l.a.createElement("div",{className:"data-entry-root",style:{top:this.props.y+"px",left:this.props.x+"px"}},l.a.createElement("input",{placeholder:"Object Name",ref:this.nameRef,defaultValue:this.props.label||""}),l.a.createElement(O,{tags:this.tags,update:function(t){return e.tags=t}}),this.props.includeSubmitButton?l.a.createElement("button",{className:"data-entry-submit",onClick:this.submit},"Submit"):null,this.props.deleteCallback?l.a.createElement("button",{className:"data-entry-delete",onClick:this.props.deleteCallback},"Delete object (\u232b)"):null)}},{key:"submit",value:function(){var e=this.nameRef.current.value.trim();0!==e.length?this.tags.length<2?alert("Please enter at least 2 descriptive tags for the object separated by enter. Examples include: color, shape, size, orientation, etc."):this.props.onSubmit({name:e.toLowerCase(),tags:this.tags}):alert("You must enter the object's name")}}]),a}(l.a.Component),I=function(e){Object(s.a)(a,e);var t=Object(r.a)(a);function a(e){var i;return Object(n.a)(this,a),(i=t.call(this,e)).onTouchEnd=function(e){i.isDrawing=!1,i.ctx.strokeStyle="red",i.ctx.lineJoin="round",i.ctx.linewidth=5,i.ctx.beginPath(),i.ctx.moveTo(i.currentMouse.x,i.currentMouse.y),i.ctx.lineTo(i.firstCoordinate.x,i.firstCoordinate.y),i.ctx.closePath(),i.ctx.stroke(),i.isDrawing=!1,i.pointList.push(i.firstCoordinate)},i.onTouchMove=function(e){if(i.isDrawing){var t=i.canvas,a=e.touches[0],n=(a.clientX-t.offsetLeft)/i.scale,s=(a.clientY-t.offsetTop)/i.scale;i.currentMouse?(i.prevMouse=i.currentMouse,i.currentMouse={x:n,y:s}):(i.currentMouse={x:n,y:s},i.firstCoordinate=i.currentMouse),i.pointList.push(i.currentMouse),i.prevMouse&&(i.ctx=i.canvas.getContext("2d"),i.ctx&&(i.ctx.strokeStyle="red",i.ctx.lineJoin="round",i.ctx.linewidth=5,i.ctx.beginPath(),i.ctx.moveTo(i.prevMouse.x,i.prevMouse.y),i.ctx.lineTo(i.currentMouse.x,i.currentMouse.y),i.ctx.closePath(),i.ctx.stroke()))}},i.canvasRef=l.a.createRef(),i.imgRef=l.a.createRef(),i.isDrawing=!1,i.annotationName="",i.annotationTags=[],i.pointList=[],i.isDrawing=!0,i}return Object(i.a)(a,[{key:"componentDidMount",value:function(){this.canvas=this.canvasRef.current,this.ctx=this.canvas.getContext("2d"),this.Offset={x:0,y:0},this.baseScale=this.canvas.width/this.props.img.width,this.scale=this.baseScale,this.canvas.height=this.props.img.height*this.baseScale,this.currentMouse=null,this.firstCoordinate=null,this.prevMouse=null,this.update()}},{key:"update",value:function(){this.ctx.resetTransform(),this.ctx.clearRect(0,0,this.canvas.width,this.canvas.height),this.ctx.setTransform(this.scale,0,0,this.scale,this.Offset.x,this.Offset.y),this.ctx.drawImage(this.props.img,0,0)}},{key:"dataEntrySubmit",value:function(e){this.drawingData=e,this.annotationTags=this.drawingData.tags,this.annotationName=this.drawingData.name,this.props.setMode("select")}},{key:"clearMask",value:function(){this.pointList=[],this.update()}},{key:"render",value:function(){var e="500px";this.props.imageWidth&&(e=this.props.imageWidth);var t=this.canvas&&this.canvas.getBoundingClientRect().left,a=this.canvas&&this.canvas.getBoundingClientRect().bottom+50;return l.a.createElement("div",null,l.a.createElement("div",null," Outline image to annotate "),l.a.createElement("button",{onClick:this.clearMask.bind(this)},"Clear Mask"),l.a.createElement("canvas",{style:{touchAction:"none"},ref:this.canvasRef,width:e,height:e,tabIndex:"0",onTouchMove:this.onTouchMove,onTouchEnd:this.onTouchEnd,onMouseUp:this.mouseEvent}),l.a.createElement(C,{x:t,y:a,onSubmit:this.dataEntrySubmit.bind(this),includeSubmitButton:!0,isMobile:this.props.isMobile}))}}]),a}(l.a.Component),_=function(e){Object(s.a)(a,e);var t=Object(r.a)(a);function a(e){var i;return Object(n.a)(this,a),(i=t.call(this,e)).state={currentMode:"select"},i.image=i.props.image,i.stateManager=i.props.stateManager,i}return Object(i.a)(a,[{key:"setMode",value:function(e){this.setState({currentMode:e})}},{key:"render",value:function(){var e=this;return"select"===this.state.currentMode?l.a.createElement("div",null,l.a.createElement("button",{onClick:function(){e.setMode("annotating")}},"Start Annotating"),l.a.createElement("img",{width:this.props.imageWidth,height:this.props.imageWidth,src:this.image.src})):l.a.createElement("div",null,l.a.createElement(I,{img:this.image,imageWidth:this.props.imageWidth,setMode:this.setMode.bind(this)}))}}]),a}(l.a.Component),R=a(138),T=a.n(R),P=function(e){Object(s.a)(a,e);var t=Object(r.a)(a);function a(e){var i;return Object(n.a)(this,a),(i=t.call(this,e)).state={currentMode:"camera",img:null,webCamPermissions:"granted",videoConstraints:{facingMode:{exact:"environment"},height:i.props.imageWidth,width:i.props.imageWidth}},i.webcamRef=l.a.createRef(),navigator.permissions&&navigator.permissions.query&&navigator.permissions.query({name:"camera"}).then((function(e){i.setState({webCamPermissions:e.state})})),i}return Object(i.a)(a,[{key:"screenshot",value:function(){var e=this.webcamRef.current.getScreenshot(),t=new Image(this.props.imageWidth,this.props.imageWidth);t.src=e,this.setState({currentMode:"annotation",img:t})}},{key:"switchCamera",value:function(){var e=this.state.videoConstraints.facingMode;e="user"===e?{exact:"environment"}:"user",this.setState({videoConstraints:{facingMode:e,height:this.props.imageWidth,width:this.props.imageWidth}})}},{key:"render",value:function(){return"camera"===this.state.currentMode?"denied"===this.state.webCamPermissions?l.a.createElement("div",null," Please grant camera permissions "):l.a.createElement("div",null,l.a.createElement(T.a,{height:this.props.imageWidth,width:this.props.imageWidth,videoConstraints:this.state.videoConstraints,ref:this.webcamRef}),l.a.createElement("button",{onClick:this.screenshot.bind(this)}," Capture "),l.a.createElement("button",{onClick:this.switchCamera.bind(this)},"Switch Camera")):"annotation"===this.state.currentMode?v.a.useDesktopComponentOnMobile?l.a.createElement(j.a,{imageWidth:this.props.imageWidth,image:this.state.img,stateManager:v.a,isMobile:!0,isFromCamera:!0}):l.a.createElement(_,{imageWidth:this.props.imageWidth-25,image:this.state.img,stateManager:v.a}):void 0}}]),a}(l.a.Component),L=function(e){Object(s.a)(a,e);var t=Object(r.a)(a);function a(e){var i;Object(n.a)(this,a);var s=window.innerWidth;return(i=t.call(this,e)).state={screen:"home",imageWidth:.4*s,screenWidth:s,objectRGB:null},i}return Object(i.a)(a,[{key:"componentDidMount",value:function(){v.a&&v.a.connect(this)}},{key:"paneHandler",value:function(e){this.setState({screen:e})}},{key:"render",value:function(){var e;return"home"===this.state.screen?e=l.a.createElement(k,{imageWidth:this.state.imageWidth}):"navigation"===this.state.screen?e=l.a.createElement(M,{imageWidth:this.state.imageWidth}):"settings"===this.state.screen?e=l.a.createElement(x,{imageWidth:this.state.imageWidth}):"annotation"===this.state.screen?e=v.a.useDesktopComponentOnMobile?l.a.createElement(j.a,{imageWidth:this.state.screenWidth-25,image:this.state.objectRGB,stateManager:v.a,isMobile:!0}):l.a.createElement(_,{imageWidth:this.state.screenWidth-25,image:this.state.objectRGB,stateManager:v.a}):"camera"===this.state.screen&&(e=l.a.createElement(P,{imageWidth:this.state.screenWidth-25})),l.a.createElement(c.a,{fluid:!0},l.a.createElement("div",{style:{paddingBottom:50}},e),l.a.createElement("div",null,l.a.createElement(d,{paneHandler:this.paneHandler.bind(this)})))}}]),a}(l.a.Component);t.a=L},9:function(e,t,a){"use strict";var n=a(17),i=a(103),s=a.n(i),r=a(133),o=a(2),l=a(3),c=a(134),h=a.n(c),u=a(43),d=a(90),p=a(22),m=a(60),g=a(34),f=a(42),v=a(70),b=a(44),y=a(56),k=a(85),E=a(83),M=a(84),w=a(89),x=a(87),j=a(88),S=a(113),O=a(71),C=new(function(){function e(){Object(o.a)(this,e),this.refs=[],this.socket=null,this.default_url="http://localhost:8000",this.connected=!1,this.initialMemoryState={objects:new Map,humans:new Map,chatResponse:{},chats:[{msg:"",failed:!1},{msg:"",failed:!1},{msg:"",failed:!1},{msg:"",failed:!1},{msg:"",failed:!1}],timelineEvent:"",timelineEventHistory:[],timelineSearchResults:[],timelineDetails:[],timelineFilters:["Perceive","Dialogue","Interpreter","Memory"],timelineSearchPattern:"",agentType:"locobot"},this.session_id=null,this.processMemoryState=this.processMemoryState.bind(this),this.setChatResponse=this.setChatResponse.bind(this),this.setConnected=this.setConnected.bind(this),this.updateAgentType=this.updateAgentType.bind(this),this.updateStateManagerMemory=this.updateStateManagerMemory.bind(this),this.keyHandler=this.keyHandler.bind(this),this.updateVoxelWorld=this.updateVoxelWorld.bind(this),this.setVoxelWorldInitialState=this.setVoxelWorldInitialState.bind(this),this.memory=this.initialMemoryState,this.processRGB=this.processRGB.bind(this),this.processDepth=this.processDepth.bind(this),this.processRGBDepth=this.processRGBDepth.bind(this),this.processObjects=this.processObjects.bind(this),this.showAssistantReply=this.showAssistantReply.bind(this),this.processHumans=this.processHumans.bind(this),this.processMap=this.processMap.bind(this),this.returnTimelineEvent=this.returnTimelineEvent.bind(this),this.onObjectAnnotationSave=this.onObjectAnnotationSave.bind(this),this.startLabelPropagation=this.startLabelPropagation.bind(this),this.labelPropagationReturn=this.labelPropagationReturn.bind(this),this.onSave=this.onSave.bind(this),this.saveAnnotations=this.saveAnnotations.bind(this),this.annotationRetrain=this.annotationRetrain.bind(this),this.goOffline=this.goOffline.bind(this),this.handleMaxFrames=this.handleMaxFrames.bind(this);var t=new URLSearchParams(window.location.search),a=t.get("turk_experiment_id"),n=t.get("mephisto_agent_id"),i=t.get("turk_worker_id");this.setTurkExperimentId(a),this.setMephistoAgentId(n),this.setTurkWorkerId(i),this.default_url=window.location.host;var s=localStorage.getItem("server_url");"undefined"!==s&&void 0!==s&&null!==s||(s=this.default_url),this.setUrl(s),this.fps_time=performance.now(),this.curFeedState={rgbImg:null,depth:null,objects:[],origObjects:[],pose:null},this.prevFeedState={rgbImg:null,depth:null,objects:[],pose:null},this.stateProcessed={rgbImg:!1,depth:!1,objects:!1,pose:!1},this.frameCount=0,this.categories=new Set,this.properties=new Set,this.annotationsSaved=!0,this.offline=!1,this.frameId=0,this.offlineObjects={},this.updateObjects=[!1,!1],this.useDesktopComponentOnMobile=!0}return Object(l.a)(e,[{key:"setDefaultUrl",value:function(){localStorage.removeItem("server_url"),this.setUrl(this.default_url)}},{key:"setUrl",value:function(e){this.url=e,localStorage.setItem("server_url",e),this.socket&&this.socket.removeAllListeners(),this.restart(this.url)}},{key:"setTurkExperimentId",value:function(e){localStorage.setItem("turk_experiment_id",e)}},{key:"getTurkExperimentId",value:function(){return localStorage.getItem("turk_experiment_id")}},{key:"setMephistoAgentId",value:function(e){localStorage.setItem("mephisto_agent_id",e)}},{key:"getMephistoAgentId",value:function(){return localStorage.getItem("mephisto_agent_id")}},{key:"setTurkWorkerId",value:function(e){localStorage.setItem("turk_worker_id",e)}},{key:"getTurkWorkerId",value:function(){return localStorage.getItem("turk_worker_id")}},{key:"restart",value:function(e){var t=this;this.socket=h.a.connect(e,{transports:["polling","websocket"]});var a=this.socket;a.on("reconnect_attempt",(function(){a.io.opts.transports=["polling","websocket"]})),a.on("connect",(function(e){var a="";function n(){return(n=Object(r.a)(s.a.mark((function e(){var t,a;return s.a.wrap((function(e){for(;;)switch(e.prev=e.next){case 0:return e.next=2,fetch("https://api.ipify.org/?format=json");case 2:return t=e.sent,e.next=5,t.json();case 5:return a=e.sent,e.abrupt("return",a);case 7:case"end":return e.stop()}}),e)})))).apply(this,arguments)}(function(){return n.apply(this,arguments)})().then((function(e){a=e.ip;var n=(+new Date).toString(36);t.session_id=a+":"+n,console.log("session id is"),console.log(t.session_id),t.socket.emit("store session id",t.session_id)})),console.log("connect event"),t.setConnected(!0),t.socket.emit("get_memory_objects"),t.socket.emit("get_agent_type")})),a.on("reconnect",(function(e){console.log("reconnect event"),t.setConnected(!0),t.socket.emit("get_memory_objects"),t.socket.emit("get_agent_type")})),a.on("disconnect",(function(e){console.log("disconnect event"),t.setConnected(!1),t.memory=t.initialMemoryState,t.refs.forEach((function(e){e instanceof M.a||(e.setState(e.initialState),e.forceUpdate())})),console.log("disconnected")})),a.on("setChatResponse",this.setChatResponse),a.on("memoryState",this.processMemoryState),a.on("updateState",this.updateStateManagerMemory),a.on("updateAgentType",this.updateAgentType),a.on("rgb",this.processRGB),a.on("depth",this.processDepth),a.on("image",this.processRGBDepth),a.on("objects",this.processObjects),a.on("updateVoxelWorldState",this.updateVoxelWorld),a.on("setVoxelWorldInitialState",this.setVoxelWorldInitialState),a.on("showAssistantReply",this.showAssistantReply),a.on("humans",this.processHumans),a.on("map",this.processMap),a.on("newTimelineEvent",this.returnTimelineEvent),a.on("labelPropagationReturn",this.labelPropagationReturn),a.on("annotationRetrain",this.annotationRetrain),a.on("saveRgbSegCallback",this.saveAnnotations),a.on("handleMaxFrames",this.handleMaxFrames)}},{key:"updateStateManagerMemory",value:function(e){this.memory=e.memory,this.refs.forEach((function(e){e.forceUpdate()}))}},{key:"updateAgentType",value:function(e){var t=this;this.memory.agentType=e.agent_type,this.refs.forEach((function(e){e instanceof O.a&&e.setState({agentType:t.memory.agentType})}))}},{key:"setConnected",value:function(e){this.connected=e,this.refs.forEach((function(t){t instanceof m.a&&t.setState({connected:e})}))}},{key:"setChatResponse",value:function(e){S.isMobile&&alert("Received text message: "+e.chat),this.memory.chats=e.allChats,this.memory.chatResponse[e.chat]=e.chatResponse,this.refs.forEach((function(t){t instanceof b.a&&t.setState({status:e.status}),t instanceof v.a&&t.forceUpdate()}))}},{key:"updateVoxelWorld",value:function(e){this.refs.forEach((function(t){t instanceof y.a&&(console.log("update Voxel World with "+e.world_state),t.setState({world_state:e.world_state,status:e.status}))}))}},{key:"setVoxelWorldInitialState",value:function(e){this.refs.forEach((function(t){t instanceof y.a&&(console.log("set Voxel World Initial state: "+e.world_state),t.setState({world_state:e.world_state,status:e.status}))}))}},{key:"showAssistantReply",value:function(e){this.refs.forEach((function(t){t instanceof b.a&&t.setState({agent_reply:e.agent_reply})}))}},{key:"updateTimeline",value:function(){this.refs.forEach((function(e){e instanceof k.a&&e.forceUpdate()}))}},{key:"returnTimelineEvent",value:function(e){this.memory.timelineEventHistory.push(e),this.memory.timelineEvent=e,this.updateTimeline()}},{key:"updateTimelineResults",value:function(){this.refs.forEach((function(e){e instanceof E.a&&e.forceUpdate()}))}},{key:"keyHandler",value:function(e){var t=[],a=[];for(var n in e){var i=e[n];n=parseInt(n),a.push(n),!0===i&&(38===n&&t.push("MOVE_FORWARD"),40===n&&t.push("MOVE_BACKWARD"),37===n&&t.push("MOVE_LEFT"),39===n&&t.push("MOVE_RIGHT"),49===n&&t.push("PAN_LEFT"),50===n&&t.push("PAN_RIGHT"),51===n&&t.push("TILT_UP"),52===n&&t.push("TILT_DOWN"))}if(t.length>0){var s={};for(var r in this.refs.forEach((function(e){e instanceof j.a&&(s=e.state)})),this.socket.emit("movement command",t,s),a)e[a[r]]=!1}}},{key:"buttonHandler",value:function(e){e.length>0&&this.socket.emit("movement command",e)}},{key:"logInteractiondata",value:function(e,t){var a={};a.session_id=this.session_id,a.mephisto_agent_id=this.getMephistoAgentId(),a.turk_worker_id=this.getTurkWorkerId(),a[e]=t,this.socket.emit("interaction data",a)}},{key:"onObjectAnnotationSave",value:function(e){var t=this,a=e.nameMap,n=e.pointMap,i=e.propertyMap,s=[];for(var r in a){for(var o=r<this.curFeedState.objects.length,l=o?this.curFeedState.objects[r].id:null,c=o?this.curFeedState.objects[r].xyz:null,h=0;h<n[r].length;)!n[r][h]||n[r][h].length<3?n[r].splice(h,1):h++;var u=n[r].map((function(e){return e.map((function(e,t){return[500*e.x,500*e.y]}))})),d=this.getNewBbox(u);s.push({label:a[r],mask:u,properties:i[r].join("\n "),type:"annotate",id:l,bbox:d,xyz:c})}this.curFeedState.objects=s,this.annotationsSaved=!1,this.refs.forEach((function(e){e instanceof g.a&&e.setState({objects:t.curFeedState.objects,updateFixup:!0})})),this.offline&&(this.offlineObjects[this.frameId]=this.curFeedState.objects)}},{key:"getNewBbox",value:function(e){for(var t=[],a=[],n=0;n<e.length;n++)for(var i=0;i<e[n].length;i++)t.push(e[n][i][0]),a.push(e[n][i][1]);var s=Math.min.apply(null,t),r=Math.max.apply(null,t);return[s,Math.min.apply(null,a),r,Math.max.apply(null,a)]}},{key:"startLabelPropagation",value:function(){var e=this,t=this.prevFeedState.objects.filter((function(e){return"annotate"===e.type}));for(var a in t){this.categories.add(t[a].label),t[a].properties.split("\n ").forEach((function(t){return e.properties.add(t)}))}if(t.length>0){var i={prevRgbImg:this.prevFeedState.rgbImg,depth:this.curFeedState.depth,prevDepth:this.prevFeedState.depth,objects:t,basePose:this.curFeedState.pose,prevBasePose:this.prevFeedState.pose};this.socket.emit("label_propagation",i)}if(!this.annotationsSaved){var s={rgb:this.prevFeedState.rgbImg,objects:t,frameCount:this.frameCount,categories:[null].concat(Object(n.a)(this.categories))};this.socket.emit("save_rgb_seg",s),this.annotationsSaved=!0}this.stateProcessed.rgbImg=!0,this.stateProcessed.depth=!0,this.stateProcessed.objects=!0,this.stateProcessed.pose=!0,this.frameCount++}},{key:"labelPropagationReturn",value:function(e){var t=this;this.refs.forEach((function(a){if(a instanceof g.a)for(var n in e){for(var i=0;i<e[n].mask.length;)!e[n].mask[i]||e[n].mask[i].length<3?e[n].mask.splice(i,1):i++;e[n].bbox=t.getNewBbox(e[n].mask),a.addObject(e[n]),t.curFeedState.objects.push(e[n])}})),this.offline&&(this.offlineObjects[this.frameId]=this.curFeedState.objects),Object.keys(e).length>0&&(this.annotationsSaved=!1)}},{key:"checkRunLabelProp",value:function(){return this.curFeedState.rgbImg&&this.curFeedState.depth&&this.curFeedState.objects.length>0&&this.curFeedState.pose&&this.prevFeedState.rgbImg&&this.prevFeedState.depth&&this.prevFeedState.objects.length>0&&this.prevFeedState.pose&&!this.stateProcessed.rgbImg&&!this.stateProcessed.depth&&!this.stateProcessed.objects&&!this.stateProcessed.pose}},{key:"onSave",value:function(){var e=this;if(console.log("saving annotations, categories, and properties"),this.offline){for(var t in this.offlineObjects){var a=this.offlineObjects[t];for(var i in a){var s=a[i];this.categories.add(s.label),s.properties.split("\n ").forEach((function(t){return e.properties.add(t)}))}}var r=[null].concat(Object(n.a)(this.categories)),o=Object(n.a)(this.properties);this.socket.emit("save_categories_properties",r,o);var l=0;for(var c in this.offlineObjects){var h=this.offlineObjects[c],u=l===Object.keys(this.offlineObjects).length-1,d={filepath:this.filepath,frameId:parseInt(c),outputId:l,objects:h,categories:r,finalFrame:u};this.socket.emit("offline_save_rgb_seg",d),l++}}else if(this.annotationsSaved)this.saveAnnotations();else{var p=this.curFeedState.objects.filter((function(e){return"annotate"===e.type}));for(var m in p){this.categories.add(p[m].label),p[m].properties.split("\n ").forEach((function(t){return e.properties.add(t)}))}var g={rgb:this.curFeedState.rgbImg,objects:p,frameCount:this.frameCount,categories:[null].concat(Object(n.a)(this.categories)),callback:!0};this.socket.emit("save_rgb_seg",g),this.annotationsSaved=!0}}},{key:"saveAnnotations",value:function(){var e=[null].concat(Object(n.a)(this.categories)),t=Object(n.a)(this.properties);this.socket.emit("save_annotations",e),this.socket.emit("save_categories_properties",e,t)}},{key:"annotationRetrain",value:function(e){console.log("retrained!"),this.refs.forEach((function(t){(t instanceof g.a||t instanceof x.a)&&t.setState({modelMetrics:e})}))}},{key:"goOffline",value:function(e){console.log("Going offline with filepath",e),this.filepath=e,this.frameId=0,this.offline=!0,this.socket.emit("get_offline_frame",{filepath:this.filepath,frameId:this.frameId}),this.socket.emit("start_offline_dashboard",e),this.refs.forEach((function(e){e instanceof g.a&&e.setState({objects:[],modelMetrics:null,offline:!0})}))}},{key:"handleMaxFrames",value:function(e){this.maxOfflineFrames=e,console.log("max frames:",e)}},{key:"offlineLabelProp",value:function(e,t){var a=this.offlineObjects[e],n={filepath:this.filepath,srcFrame:e,curFrame:t,objects:a};this.socket.emit("offline_label_propagation",n)}},{key:"previousFrame",value:function(){var e=this;if(0!==this.frameId){this.frameId--,console.log("Prev frame",this.frameId),this.socket.emit("get_offline_frame",{filepath:this.filepath,frameId:this.frameId}),this.curFeedState.objects=this.offlineObjects[this.frameId]||[],this.refs.forEach((function(t){t instanceof g.a&&t.setState({objects:e.curFeedState.objects})}));var t=this.offlineObjects[this.frameId]&&this.offlineObjects[this.frameId].length>0;this.offlineObjects[this.frameId+1]&&!t&&this.offlineLabelProp(this.frameId+1,this.frameId)}else console.log("no frames under 0")}},{key:"nextFrame",value:function(){var e=this;if(this.frameId!==this.maxOfflineFrames){this.frameId++,console.log("Next frame",this.frameId),this.socket.emit("get_offline_frame",{filepath:this.filepath,frameId:this.frameId}),this.curFeedState.objects=this.offlineObjects[this.frameId]||[],this.refs.forEach((function(t){t instanceof g.a&&t.setState({objects:e.curFeedState.objects})}));var t=this.offlineObjects[this.frameId]&&this.offlineObjects[this.frameId].length>0;this.offlineObjects[this.frameId-1]&&!t&&this.offlineLabelProp(this.frameId-1,this.frameId)}else console.log("no frames over",this.maxOfflineFrames)}},{key:"processMemoryState",value:function(e){this.refs.forEach((function(t){t instanceof d.a&&t.setState({isLoaded:!0,memory:e})}))}},{key:"processRGB",value:function(e){var t=this;if(this.curFeedState.rgbImg!==e){var a=new Image;a.src="data:image/webp;base64,"+e,this.refs.forEach((function(e){e instanceof p.a&&"rgb"===e.props.type&&e.setState({isLoaded:!0,rgb:a}),t.offline&&e instanceof g.a&&e.setState({rgb:a})})),this.prevFeedState.rgbImg=this.curFeedState.rgbImg,this.curFeedState.rgbImg=e,this.stateProcessed.rgbImg=!1,this.updateObjects=[!0,!0]}this.checkRunLabelProp()&&this.startLabelPropagation()}},{key:"processDepth",value:function(e){if(this.curFeedState.depth!==e){var t=new Image;t.src="data:image/webp;base64,"+e.depthImg,this.refs.forEach((function(e){e instanceof p.a&&"depth"===e.props.type&&e.setState({isLoaded:!0,depth:t})})),this.prevFeedState.depth=this.curFeedState.depth,this.curFeedState.depth={depthImg:e.depthImg,depthMax:e.depthMax,depthMin:e.depthMin},this.stateProcessed.depth=!1}this.checkRunLabelProp()&&this.startLabelPropagation()}},{key:"processRGBDepth",value:function(e){this.processRGB(e.rgb),this.processDepth(e.depth)}},{key:"processObjects",value:function(e){if(-1!==e.image&&void 0!==e.image){var t=new Image;t.src="data:image/webp;base64,"+e.image.rgb;for(var a=0;a<e.objects.length;){for(var n=0;n<e.objects[a].mask.length;)!e.objects[a].mask[n]||e.objects[a].mask[n].length<3?e.objects[a].mask.splice(n,1):n++;0!=e.objects[a].mask.length?a++:e.objects.splice(a,1)}e.objects.forEach((function(e){e.type="detector"})),this.updateObjects[1]&&(this.prevFeedState.objects=this.curFeedState.objects,this.curFeedState.objects=JSON.parse(JSON.stringify(e.objects)),this.curFeedState.origObjects=JSON.parse(JSON.stringify(e.objects)),this.stateProcessed.objects=!1,this.updateObjects=[!1,!1],this.refs.forEach((function(a){a instanceof g.a?a.setState({objects:e.objects,rgb:t,height:e.height,width:e.width,scale:e.scale}):a instanceof w.a&&a.setState({objectRGB:t})}))),this.updateObjects[0]&&(this.updateObjects[1]=!0),this.checkRunLabelProp()&&this.startLabelPropagation()}}},{key:"processHumans",value:function(e){if(-1!==e.image&&void 0!==e.image){var t=new Image;t.src="data:image/webp;base64,"+e.image.rgb,this.refs.forEach((function(a){a instanceof f.a&&a.setState({isLoaded:!0,humans:e.humans,rgb:t,height:e.height,width:e.width,scale:e.scale})}))}}},{key:"processMap",value:function(e){var t=this;this.refs.forEach((function(a){a instanceof u.a&&a.setState({isLoaded:!0,memory:t.memory,bot_xyz:[e.x,e.y,e.yaw],obstacle_map:e.map})})),this.curFeedState.pose&&(!e||e.x===this.curFeedState.pose.x&&e.y===this.curFeedState.pose.y&&e.yaw===this.curFeedState.pose.yaw)||(this.prevFeedState.pose=this.curFeedState.pose,this.curFeedState.pose={x:e.x,y:e.y,yaw:e.yaw},this.stateProcessed.pose=!1),this.checkRunLabelProp()&&this.startLabelPropagation()}},{key:"connect",value:function(e){this.refs.push(e)}},{key:"disconnect",value:function(e){this.refs=this.refs.filter((function(t){return t!==e}))}}]),e}());t.a=C},90:function(e,t,a){"use strict";var n=a(2),i=a(3),s=a(6),r=a(5),o=a(4),l=a(0),c=a.n(l),h=a(40),u=a(55),d=a(68),p=a(52),m=a(11),g=a(15),f=a(462),v=a(78),b=["height","width","classes","columns","rowHeight","headerHeight"],y=["dataKey"],k=function(e){Object(r.a)(a,e);var t=Object(o.a)(a);function a(){var e;Object(n.a)(this,a);for(var i=arguments.length,s=new Array(i),r=0;r<i;r++)s[r]=arguments[r];return(e=t.call.apply(t,[this].concat(s))).getRowClassName=function(t){var a=t.index,n=e.props,i=n.classes,s=n.onRowClick;return Object(m.a)(i.tableRow,i.flexContainer,Object(p.a)({},i.tableRowHover,-1!==a&&null!=s))},e.cellRenderer=function(t){var a=t.cellData,n=t.columnIndex,i=e.props,s=i.columns,r=i.classes,o=i.rowHeight,l=i.onRowClick;return c.a.createElement(f.a,{component:"div",className:Object(m.a)(r.tableCell,r.flexContainer,Object(p.a)({},r.noClick,null==l)),variant:"body",style:{height:o},align:null!=n&&s[n].numeric?"right":"left"},a)},e.headerRenderer=function(t){var a=t.label,n=t.columnIndex,i=e.props,s=i.headerHeight,r=i.columns,o=i.classes;return c.a.createElement(f.a,{component:"div",className:Object(m.a)(o.tableCell,o.flexContainer,o.noClick),variant:"head",style:{height:s},align:r[n].numeric?"right":"left"},c.a.createElement("span",null,a))},e}return Object(i.a)(a,[{key:"render",value:function(){var e=this,t=this.props,a=t.height,n=t.width,i=t.classes,s=t.columns,r=t.rowHeight,o=t.headerHeight,l=Object(d.a)(t,b);return c.a.createElement(v.c,Object.assign({height:a,width:n,rowHeight:r,gridStyle:{direction:"inherit"},size:"small",headerHeight:o,className:i.table},l,{rowClassName:this.getRowClassName}),s.map((function(t,a){var n=t.dataKey,s=Object(d.a)(t,y);return c.a.createElement(v.b,Object.assign({key:n,headerRenderer:function(t){return e.headerRenderer(Object(u.a)(Object(u.a)({},t),{},{columnIndex:a}))},className:i.flexContainer,cellRenderer:e.cellRenderer,dataKey:n},s))})))}}]),a}(c.a.PureComponent);k.defaultProps={headerHeight:48,rowHeight:36};var E=Object(g.a)((function(e){return{flexContainer:{display:"flex",alignItems:"center",boxSizing:"border-box"},table:{"& .ReactVirtualized__Table__headerRow":{flip:!1,paddingRight:"rtl"===e.direction?"0 !important":void 0}},tableRow:{cursor:"pointer"},tableRowHover:{"&:hover":{backgroundColor:e.palette.action.hover}},tableCell:{flex:1},noClick:{cursor:"initial"}}}))(k);function M(e){var t=e.memoryManager,a=e.onShowMemoryDetail,n=Object(h.a)(t,2),i=n[0],s=n[1];return c.a.createElement(v.a,null,(function(e){var t=e.height,n=e.width;return c.a.createElement(E,{height:t,width:n,rowCount:i(),rowGetter:function(e){var t,a=e.index;return{uuid:(t=s(a)).data[0],id:t.data[1],type:t.data[9],name:t.data[7]}},onRowClick:function(e){e.event,e.index;var t=e.rowData;t&&t.uuid&&a(t.uuid)},columns:[{flexGrow:1,width:1,label:"ID",dataKey:"id",numeric:!0},{flexGrow:5,width:5,label:"Type",dataKey:"type",numeric:!1},{flexGrow:5,width:5,label:"Name",dataKey:"name",numeric:!1}]})}))}var w=a(135);function x(e,t){e.memories,e.named_abstractions;var a=e.reference_objects,n=e.triples,i=new Map(a.map((function(e){return[e[0],{data:e}]})));n&&n.forEach((function(e){var t=e[1],a=i.get(t);a&&(a.triples?a.triples.push(e):a.triples=[e])}));var s=[];if(null!=t&&t.length>0){var r,o=Object(w.a)(i.values());try{for(o.s();!(r=o.n()).done;){var l=r.value;JSON.stringify(l).includes(t)&&s.push(l)}}catch(c){o.e(c)}finally{o.f()}}else s=Array.from(i.values());return[function(){return s.length},function(e){return s[e]},function(e){return i.get(e)}]}var j=a(86),S=a(463),O=a(464),C=a(415),I=a(136),_=a.n(I),R=a(451),T=a(276),P=a(449),L=function(e){Object(r.a)(a,e);var t=Object(o.a)(a);function a(e){var i;return Object(n.a)(this,a),(i=t.call(this,e)).initialState={height:400,width:600,isLoaded:!1,memory:null,filter:"",showDetail:!1,detailUUID:null},i.state=i.initialState,i.outer_div=c.a.createRef(),i.resizeHandler=i.resizeHandler.bind(Object(s.a)(i)),i}return Object(i.a)(a,[{key:"resizeHandler",value:function(){if(null!=this.outer_div&&null!=this.outer_div.current){var e=this.outer_div.current,t=e.clientHeight,a=e.clientWidth;(void 0!==t&&t!==this.state.height||void 0!==a&&a!==this.state.width)&&this.setState({height:t,width:a})}}},{key:"componentDidMount",value:function(){this.props.stateManager&&this.props.stateManager.connect(this),void 0!==this.props.glContainer&&this.props.glContainer.on("resize",this.resizeHandler)}},{key:"componentDidUpdate",value:function(e,t){this.resizeHandler()}},{key:"render",value:function(){var e=this;if(!this.state.isLoaded)return c.a.createElement("p",null,"Loading");var t=this.state,a=t.height,n=t.width,i=t.memory,s=t.isLoaded;if(0===a&&0===n)return c.a.createElement("div",{ref:this.outer_div,style:{height:"100%",width:"100%"}});if(!s)return c.a.createElement("div",{ref:this.outer_div,style:{height:"100%",width:"100%"}},"Loading...");var r=Object(T.a)({palette:{type:"dark"}}),o=new x(i,this.state.filter),l=this.state.height-24,h=this.state.width-24,u=function(){e.setState({showDetail:!1})};return c.a.createElement(P.a,{theme:r},c.a.createElement(S.a,{style:{borderBlockColor:"white"},color:"primary",id:"outlined-uncontrolled",label:"Search",margin:"dense",variant:"outlined",onChange:function(t){e.setState({filter:t.target.value})}}),c.a.createElement(M,{height:l,width:h,memoryManager:o,onShowMemoryDetail:function(t){e.setState({detailUUID:t,showDetail:!0})}}),c.a.createElement(O.a,{anchor:"right",open:this.state.showDetail,onClose:function(){u()}},c.a.createElement("div",{style:{width:450}},c.a.createElement(C.a,{onClick:function(){return u()}},c.a.createElement(_.a,null)),c.a.createElement(R.a,null),c.a.createElement(j.a,{memoryManager:o,uuid:this.state.detailUUID}))))}}]),a}(c.a.Component);t.a=L}},[[410,157,0,1]]]);