anchor.diff:  -author('mats cronqvist').
   -export([sax/3
            , unit/0,unit/1
  -         , file_parse/1
            , wget_parse/1
            , wget_print/1]).
   
  
positive.code: -define(ev(X), ?ws(X); X==$>; X==$=).
  
  tokenize(Str) when is_integer(hd(Str))-> tokenize(list_to_binary(Str));
  tokenize(Str) when is_binary(Str) -> tokenize({text,Str,{text,""}});
  
  tokenize({text,Str,Token}) ->
    case ff(text,Str) of
      {{tag,""},EStr,{text,Txt}} -> 
        try [Token,{text,Txt}]++tokenize(t({tag,""},ws(EStr)))
        catch _:_ -> TB = list_to_binary(Txt),
                     tokenize({text,<<TB/binary,"&lt;",EStr/binary>>,Token})
        end;
      {{text,Txt},"",eof} ->
        [Token,{eof,{text,Txt}}]
    end;
  tokenize({TZ,Str,Token}) ->
    [{Token,{TZ,Str}}].
  
  t({tag,""},?m("!--",Str))                -> t({comm,""},Str);
  t({tag,""},?m("!",Str))                  -> t({'!',""},ws(Str));
  t({tag,""},?m("?",Str))                  -> t({que,""},ws(Str));
  t({tag,""},?m("/",Str))                  -> t({end_tag,""},ws(Str));
  t({tag,Tag},?D(X,S))when ?ev(X)          -> t({attr,"",{Tag,[]}},ws(S));
  t({tag,Tag},?d(X,Str))when ?ok(X)        -> t({tag,Tag++[X]},Str);
  
  t({'!',DT},?m(">",Str))                  -> {text,Str,{'!',dc(DT)}};
  t({'!',DT},?d(X,Str))                    -> t({'!',DT++[X]},Str);
  
  t({que,DT},?m("?>",Str))                 -> {text,Str,{'?',dc(DT)}};
  t({que,DT},?d(X,Str))                    -> t({que,DT++[X]},Str);
  
  t({comm,Comm},?m("-->",Str))             -> {text,Str,{comment,Comm}};
  t({comm,Comm},?d(X,Str))                 -> t({comm,Comm++[X]},Str);
  
  t({etag,Tag,Attrs},?m("/>",Str))         -> {text,Str,{sc,Tag,Attrs}};
  t({etag,"script",Attrs},?m(">",Str))     -> {script,Str,{open,"script",Attrs}};
  t({etag,"style",Attrs},?m(">",Str))      -> {style,Str,{open,"style",Attrs}};
  t({etag,Tag,Attrs},?m(">",Str))          -> {text,Str,{open,Tag,Attrs}};
  
  t({end_tag,Tag},?D(X,S))when ?ev(X)      -> t({end_tag,Tag,'>'},ws(S));
  t({end_tag,Tag,'>'},?m(">",Str))         -> {text,Str,{close,dc(Tag)}};
  t({end_tag,Tag},?d(X,Str))               -> t({end_tag,Tag++[X]},Str);
  
  t({attr,"",{Tag,As}},?D(X,S))when ?ev(X) -> t({etag,dc(Tag),As},S);
  t({attr,"",{Tag,As}},?M("/>",S))         -> t({etag,dc(Tag),As},S);
  t({attr,A,{T,As}},?D(X,S))when ?ev(X)    -> t({eatt,{dc(A),T,As}},ws(S));
  t({attr,A,{T,As}},?M("/>",S))            -> t({eatt,{dc(A),T,As}},ws(S));
  t({attr,A,TAs},?d(X,Str))when ?ok(X)     -> t({attr,A++[X],TAs},Str);
  
  t({eatt,ATAs},?m("=",Str))               -> t({val,ATAs},ws(Str));
  t({eatt,{A,T,As}},S)                     -> t({attr,"",{T,As++[{A,""}]}},ws(S));
  
  t({val,ATAs},?m("'",Str))                -> t({sqval,"",ATAs},Str);%singlequ
  t({val,ATAs},?m("\"",Str))               -> t({dqval,"",ATAs},Str);%doublequ
  t({val,ATAs},Str)                        -> t({uqval,"",ATAs},Str);%unquoted
  
  t({sqval,V,{A,T,As}},?m("'",S))          -> t({attr,"",{T,As++[{A,V}]}},ws(S));
  t({sqval,V,ATAs},?d(X,Str))              -> t({sqval,V++[X],ATAs},Str);
  
  t({dqval,V,{A,T,As}},?m("\"",S))         -> t({attr,"",{T,As++[{A,V}]}},ws(S));
  t({dqval,V,ATAs},?d(X,Str))              -> t({dqval,V++[X],ATAs},Str);
  
  t({uqval,V,{A,T,As}},?D(X,S))when ?ev(X) -> t({attr,"",{T,As++[{A,V}]}},ws(S));
  t({uqval,V,{A,T,As}},?M("/>",S))         -> t({attr,"",{T,As++[{A,V}]}},ws(S));
  t({uqval,V,ATAs},?d(X,Str))              -> t({uqval,V++[X],ATAs},Str);
  
  t(script,Str)                            -> ff(script,Str);
  t(style,Str)                             -> ff(style,Str);
  t(text,Str)                              -> ff(text,Str);
  
  t(X,"")                                  -> {X,"",eof}.
  
  ff(What,Str) -> ff(What,Str,[]).
  
  ff(_Wht,<<>>,A)        -> {{text,lists:reverse(A)},"",eof};
  ff(_Wht,?m("<",Str),A) -> {{tag,""},Str,{text,lists:reverse(A)}};
  ff(What,?d(H,Str),A)   -> ff(What,Str,[H|A]).
  
  % ff(What,Str) ->
  %   case ff(Str,ff(What),[1,2]) of
  %     [Scr,Rest] -> {{tag,""},ws(Rest),{text,Scr}};
  %     [] -> {{text,Str},"",eof}
  %   end.
  
  % ff(script)-> "(.*)<(\\s*/\\s*script\\s*>.*)\$";
  % ff(style) -> "(.*)<(\\s*/\\s*style\\s*>.*)\$";
  % ff(text)  -> "(.*)<(.*)\$".
  
  % ff(Str,P,Groups) ->
  %   case re:run(Str,P,[{capture,Groups,binary},dotall,caseless,ungreedy]) of
  %     {match,Ms} -> Ms;
  %     nomatch -> []
  %  end.
  
  ws(?d(X,Str)) when ?ws(X) -> ws(Str);
  ws(Str) -> Str.
  
negative.code: 
  %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  %% Parsing real pages
  wget(Url) ->
    inets:start(),
    {ok, {_Rc, _Hdrs, Body}} = http:request(get, {Url, []}, [], []),
    Body.
  
  wget_parse(Url) ->
    sax(wget(Url),fun(T,A)-> A++[T] end, []).
  
  wget_print(Url) ->
    io:fwrite("~s~n",[wget(Url)]).
  
  %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  %% ad-hoc unit test
  