title: Saving BeatifulSoup output to mongo and loading it back again
body: <p>I have a crawler, that obtains certain web pages for my app.
  I want to separate concerns, the crawler should be 'dumb', just fetch the page, take the BeautifulSoup JSON, and save it into MongoDB.</p>
  
  <p>Other workers should then read the MongoDB documents and extract the 
  relevant information into a relational model.</p>
  
  <p>The question is how to safely convert a BeautifulSoup object to JSON (MongoDB document) and back to it self, safely and without errors.</p>
  
  <p>Edit:
  <strong>Illustration</strong></p>
  
  <pre><code> import urllib2
   import json
   from bs4 import BeautifulSoup
   req = urllib2.Request('http://www.google.com')
   res = urllib2.urlopen(req)
   soup = BeautifulSoup(res.read())
   content = soup.findAll(text=True)
   soup_json = json.dumps(content)
   soup_json
  </code></pre>
  
  <p>Outputs:</p>
  
  <pre><code>'["doctype html", "Google", "(function(){\\nwindow.google={kEI:\\"LGktU9bfHqHk4wT1poGoAg\\",getEI:function(a){for(var b;a&amp;&amp;(!a.getAttribute||!(b=a.getAttribute(\\"eid\\")));)a=a.parentNode;return b||google.kEI},https:function(){return\\"https:\\"==window.location.protocol},kEXPI:\\"4006,17259,4000116,4007661,4007830,4008067,4008133,4008142,4009033,4009352,4009565,4009641,4010297,4010806,4010858,4010899,4011228,4011258,4011679,4011959,4012373,4012504,4012507,4013338,4013374,4013414,4013416,4013591,4013723,4013747,4013787,4013823,4013967,4013979,4014016,4014431,4014515,4014636,4014649,4014671,4014792,4014804,4014813,4014991,4015119,4015155,4015195,4015234,4015260,4015320,4015444,4015497,4015514,4015582,4015589,4015637,4015638,4015640,4015690,4015772,4015853,4015904,4015991,4015995,4016007,4016047,4016062,4016139,4016167,4016193,4016304,4016311,4016407,8300007,8300015,8300018,8500149,8500157,10200002,10200012,10200029,10200030,10200040,10200045,10200048,10200053,10200055,10200066,10200083,10200103,10200120,10200134,10200157\\",kCSI:{e:\\"4006,17259,4000116,4007661,4007830,4008067,4008133,4008142,4009033,4009352,4009565,4009641,4010297,4010806,4010858,4010899,4011228,4011258,4011679,4011959,4012373,4012504,4012507,4013338,4013374,4013414,4013416,4013591,4013723,4013747,4013787,4013823,4013967,4013979,4014016,4014431,4014515,4014636,4014649,4014671,4014792,4014804,4014813,4014991,4015119,4015155,4015195,4015234,4015260,4015320,4015444,4015497,4015514,4015582,4015589,4015637,4015638,4015640,4015690,4015772,4015853,4015904,4015991,4015995,4016007,4016047,4016062,4016139,4016167,4016193,4016304,4016311,4016407,8300007,8300015,8300018,8500149,8500157,10200002,10200012,10200029,10200030,10200040,10200045,10200048,10200053,10200055,10200066,10200083,10200103,10200120,10200134,10200157\\",ei:\\"LGktU9bfHqHk4wT1poGoAg\\"},authuser:0,ml:function(){},kHL:\\"iw\\",time:function(){return(new Date).getTime()},log:function(a,b,c,h,k){var d=\\nnew Image,f=google.lc,e=google.li,g=\\"\\";d.onerror=d.onload=d.onabort=function(){delete f[e]};f[e]=d;c||-1!=b.search(\\"&amp;ei=\\")||(g=\\"&amp;ei=\\"+google.getEI(h));c=c||\\"/\\"+(k||\\"gen_204\\")+\\"?atyp=i&amp;ct=\\"+a+\\"&amp;cad=\\"+b+g+\\"&amp;zx=\\"+google.time();a=/^http:/i;a.test(c)&amp;&amp;google.https()?(google.ml(Error(\\"GLMM\\"),!1,{src:c}),delete f[e]):(d.src=c,google.li=e+1)},lc:[],li:0,y:{},x:function(a,b){google.y[a.id]=[a,b];return!1},load:function(a,b,c){google.x({id:a+l++},function(){google.load(a,b,c)})}};var l=0;})();\\n(function(){google.sn=\\"webhp\\";google.timers={};google.startTick=function(a,b){google.timers[a]={t:{start:google.time()},bfr:!!b}};google.tick=function(a,b,g){google.timers[a]||google.startTick(a);google.timers[a].t[b]=g||google.time()};google.startTick(\\"load\\",!0);\\ntry{}catch(d){}})();\\nvar _gjwl=location;function _gjuc(){var a=_gjwl.href.indexOf(\\"#\\");if(0&lt;=a&amp;&amp;(a=_gjwl.href.substring(a),0&lt;a.indexOf(\\"&amp;q=\\")||0&lt;=a.indexOf(\\"#q=\\"))&amp;&amp;(a=a.substring(1),-1==a.indexOf(\\"#\\"))){for(var d=0;d&lt;a.length;){var b=d;\\"&amp;\\"==a.charAt(b)&amp;&amp;++b;var c=a.indexOf(\\"&amp;\\",b);-1==c&amp;&amp;(c=a.length);b=a.substring(b,c);if(0==b.indexOf(\\"fp=\\"))a=a.substring(0,d)+a.substring(c,a.length),c=d;else if(\\"cad=h\\"==b)return 0;d=c}_gjwl.href=\\"/search?\\"+a+\\"&amp;cad=h\\";return 1}return 0}\\nfunction _gjh(){!_gjuc()&amp;&amp;window.google&amp;&amp;google.x&amp;&amp;google.x({id:\\"GJH\\"},function(){google.nav&amp;&amp;google.nav.gjh&amp;&amp;google.nav.gjh()})};\\nwindow._gjh&amp;&amp;_gjh();", "#gbar,#guser{font-size:13px;padding-top:1px !important;}#gbar{height:22px}#guser{padding-bottom:7px !important;text-align:left}.gbh,.gbd{border-top:1px solid #c9d7f1;font-size:1px}.gbh{height:0;position:absolute;top:24px;width:100%}@media all{.gb1{height:22px;margin-left:.5em;vertical-align:top}#gbar{float:right}}a.gb1,a.gb4{text-decoration:underline !important}a.gb1,a.gb4{color:#00c !important}.gbi .gb4{color:#dd8e27 !important}.gbf .gb4{color:#900 !important}", "body,td,a,p,.h{font-family:arial,sans-serif}body{margin:0;overflow-y:scroll}#gog{padding:3px 8px 0}td{line-height:.8em}.gac_m td{line-height:17px}form{margin-bottom:20px}.h{color:#36c}.q{color:#00c}.ts td{padding:0}.ts{border-collapse:collapse}em{font-weight:bold;font-style:normal}.lst{height:25px;width:496px}.gsfi,.lst{font:18px arial,sans-serif}.gsfs{font:17px arial,sans-serif}.ds{display:inline-box;display:inline-block;margin:3px 0 4px;margin-right:4px}input{font-family:inherit}a.gb1,a.gb2,a.gb3,a.gb4{color:#11c !important}body{background:#fff;color:black}a{color:#11c;text-decoration:none}a:hover,a:active{text-decoration:underline}.fl a{color:#36c}a:visited{color:#551a8b}a.gb1,a.gb4{text-decoration:underline}a.gb3:hover{text-decoration:none}#ghead a.gb2:hover{color:#fff !important}.sblc{padding-top:5px}.sblc a{display:block;margin:2px 0;margin-right:13px;font-size:11px}.lsbb{background:#eee;border:solid 1px;border-color:#ccc #ccc #999 #999;height:30px}.lsbb{display:block}.ftl,#fll a{display:inline-block;margin:0 12px}.lsb{background:url(/images/srpr/nav_logo80.png) 0 -258px repeat-x;border:none;color:#000;cursor:pointer;height:30px;margin:0;outline:0;font:15px arial,sans-serif;vertical-align:top}.lsb:active{background:#ccc}.lst:focus{outline:none}#addlang a{padding:0 3px}.tiah{width:458px}", "(function(){var src=\'/images/nav_logo176.png\';var iesg=false;document.body.onload = function(){window.n &amp;&amp; window.n();if (document.images){new Image().src=src;}\\nif (!iesg){document.f&amp;&amp;document.f.q.focus();document.gbqf&amp;&amp;document.gbqf.q.focus();}\\n}\\n})();", " ", "\\u00e7\\u00e9\\u00f4\\u00e5\\u00f9", " ", "\\u00fa\\u00ee\\u00e5\\u00f0\\u00e5\\u00fa", " ", "\\u00ee\\u00f4\\u00e5\\u00fa", " ", "YouTube", " ", "\\u00e7\\u00e3\\u00f9\\u00e5\\u00fa", " ", "Gmail", " ", "Drive", " ", "\\u00e9\\u00e5\\u00ee\\u00ef", " ", "\\u00f2\\u00e5\\u00e3", " \\u00bb", "\\u00e4\\u00e9\\u00f1\\u00e8\\u00e5\\u00f8\\u00e9\\u00e9\\u00fa \\u00e0\\u00fa\\u00f8\\u00e9\\u00ed", " | ", "\\u00e4\\u00e2\\u00e3\\u00f8\\u00e5\\u00fa", " | ", "\\u00e4\\u00e9\\u00eb\\u00f0\\u00f1", " ", "\\u00e9\\u00f9\\u00f8\\u00e0\\u00ec", "\\u00a0", "\\u00e7\\u00e9\\u00f4\\u00e5\\u00f9 \\u00ee\\u00fa\\u00f7\\u00e3\\u00ed", "\\u00eb\\u00ec\\u00e9 \\u00f9\\u00f4\\u00e4", "Google.co.il \\u00e2\\u00ed \\u00e1: ", "\\u0627\\u0644\\u0639\\u0631\\u0628\\u064a\\u0629", " ", "English", " \\u00f4\\u00f8\\u00f1\\u00e5\\u00ed \\u00e1-Google", "\\u00f4\\u00fa\\u00f8\\u00e5\\u00f0\\u00e5\\u00fa \\u00f2\\u00f1\\u00f7\\u00e9\\u00e9\\u00ed", "\\u00e4\\u00eb\\u00ec \\u00e0\\u00e5\\u00e3\\u00e5\\u00fa Google", "Google.com", "\\u00a9 2013 - ", "\\u00f4\\u00f8\\u00e8\\u00e9\\u00e5\\u00fa \\u00e5\\u00fa\\u00f0\\u00e0\\u00e9\\u00ed", "if(google.y)google.y.first=[];(function(){function b(a){window.setTimeout(function(){var c=document.createElement(\\"script\\");c.src=a;document.getElementById(\\"xjsd\\").appendChild(c)},0)}google.dljp=function(a){google.xjsu=a;b(a)};google.dlj=b;})();\\nif(!google.xjs){window._=window._||{};window._._DumpException=function(e){throw e};if(google.timers&amp;&amp;google.timers.load.t){google.timers.load.t.xjsls=new Date().getTime();}google.dljp(\'/xjs/_/js/k\\\\x3dxjs.hp.en_US.X67G-1Nbjpc.O/m\\\\x3dsb_he,pcc/rt\\\\x3dj/d\\\\x3d1/sv\\\\x3d1/rs\\\\x3dAItRSTO_vkVhEK6twEUdYclvmSrFcRL-Zw\');google.xjs=1;}google.pmc={\\"sb_he\\":{\\"agen\\":true,\\"cgen\\":true,\\"client\\":\\"heirloom-hp\\",\\"dh\\":true,\\"ds\\":\\"\\",\\"eqch\\":true,\\"fl\\":true,\\"host\\":\\"google.co.il\\",\\"jsonp\\":true,\\"msgs\\":{\\"dym\\":\\"\\u00e4\\u00e0\\u00ed \\u00e4\\u00fa\\u00eb\\u00e5\\u00e5\\u00f0\\u00fa \\u00ec:\\",\\"lcky\\":\\"\\u00e9\\u00e5\\u00fa\\u00f8 \\u00ee\\u00e6\\u00ec \\u00ee\\u00f9\\u00eb\\u00ec\\",\\"lml\\":\\"\\u00ec\\u00ee\\u00e9\\u00e3\\u00f2 \\u00f0\\u00e5\\u00f1\\u00f3\\",\\"oskt\\":\\"\\u00eb\\u00ec\\u00e9 \\u00e4\\u00e6\\u00f0\\u00e4\\",\\"psrc\\":\\"\\u00e7\\u00e9\\u00f4\\u00e5\\u00f9 \\u00e6\\u00e4 \\u00e4\\u00e5\\u00f1\\u00f8 \\u00ee\\\\u003Ca href=\\\\\\"/history\\\\\\"\\\\u003E\\u00e4\\u00e9\\u00f1\\u00e8\\u00e5\\u00f8\\u00e9\\u00e9\\u00fa \\u00e4\\u00e0\\u00e9\\u00f0\\u00e8\\u00f8\\u00f0\\u00e8\\\\u003C/a\\\\u003E \\u00f9\\u00ec\\u00ea\\",\\"psrl\\":\\"\\u00e4\\u00f1\\u00f8\\",\\"sbit\\":\\"\\u00e7\\u00f4\\u00f9 \\u00ec\\u00f4\\u00e9 \\u00fa\\u00ee\\u00e5\\u00f0\\u00e4\\",\\"srch\\":\\"\\u00e7\\u00e9\\u00f4\\u00e5\\u00f9 \\u00e1-Google\\"},\\"ovr\\":{},\\"pq\\":\\"\\",\\"qcpw\\":false,\\"scd\\":10,\\"sce\\":5,\\"stok\\":\\"AVgtYJUWkObPx6V5QqvD7hitdNE\\"},\\"pcc\\":{}};google.y.first.push(function(){if(google.med){google.med(\'init\');google.initHistory();google.med(\'history\');}});if(google.j&amp;&amp;google.j.en&amp;&amp;google.j.xi){window.setTimeout(google.j.xi,0);}", "(function(){if(google.timers&amp;&amp;google.timers.load.t){var b,c,d,e,g=function(a,f){a.removeEventListener?(a.removeEventListener(\\"load\\",f,!1),a.removeEventListener(\\"error\\",f,!1)):(a.detachEvent(\\"onload\\",f),a.detachEvent(\\"onerror\\",f))},h=function(a){e=(new Date).getTime();++c;a=a||window.event;a=a.target||a.srcElement;g(a,h)},k=document.getElementsByTagName(\\"img\\");b=k.length;for(var l=c=0,m;l&lt;b;++l)m=k[l],m.complete||\\"string\\"!=typeof m.src||!m.src?++c:m.addEventListener?(m.addEventListener(\\"load\\",h,!1),m.addEventListener(\\"error\\",\\nh,!1)):(m.attachEvent(\\"onload\\",h),m.attachEvent(\\"onerror\\",h));d=b-c;var n=function(){if(google.timers.load.t){google.timers.load.t.ol=(new Date).getTime();google.timers.load.t.iml=e;google.kCSI.imc=c;google.kCSI.imn=b;google.kCSI.imp=d;void 0!==google.stt&amp;&amp;(google.kCSI.stt=google.stt);google.csiReport&amp;&amp;google.csiReport()}};window.addEventListener?window.addEventListener(\\"load\\",n,!1):window.attachEvent&amp;&amp;\\nwindow.attachEvent(\\"onload\\",n);google.timers.load.t.prt=e=(new Date).getTime()};})();\\n"]'
  </code></pre>
  
  <p>This JSON should be saved in MongoDB, in a manner that will allow me to respote a Beatiful soup object from it later.</p>