anchor.diff:  	 get_client_files/2, file_format/1, tokenize/3,
   	 concat_toks/1, test_framework_used/1,
   	 get_modules_by_file/1]).
  -
  --export([add_category/1]).
  -
   
   -include("../include/wrangler.hrl").
   
  
positive.code: %% =====================================================================
  % @doc Attach syntax category information to AST nodes.
  %% =====================================================================
  
  %%-spec(add_category(Node::syntaxTree()) -> syntaxTree()).
  add_category(Node) ->
      case refac_syntax:type(Node) of
        form_list ->
  	  Es = refac_syntax:form_list_elements(Node),
  	  Es1 = lists:map(fun (E) -> add_category(E) end, Es),
  	  Node1 = rewrite(Node, refac_syntax:form_list(Es1)),
  	  refac_syntax:add_ann({category, form_list}, Node1);
        attribute -> add_category(Node, attribute);
        function -> add_category(Node, function);
        rule -> add_category(Node, rule);
        error_marker -> add_category(Node, error_marker);
        warning_marker -> add_category(Node, warning_marker);
        eof_marker -> add_category(Node, eof_marker);
        comment -> add_category(Node, comment);
        %%  macro -> add_category(Node, macro);
        _ -> add_category(Node, unknown)
      end.
  
  add_category(Node, C) -> {Node1, _} = ast_traverse_api:stop_tdTP(fun do_add_category/2, Node, C),
  			 Node1.
  do_add_category(Node, C) ->
      if is_list(Node) -> {lists:map(fun (E) -> add_category(E, C) end, Node), true};
         true ->
  	   case refac_syntax:type(Node) of
  	     clause ->
  		 B = refac_syntax:clause_body(Node),
  		 P = refac_syntax:clause_patterns(Node),
  		 G = refac_syntax:clause_guard(Node),
  		 B1 = add_category(B, expression),
  		 P1 = add_category(P, pattern),
  		 G1 = case G of
  			none -> none;
  			_ -> add_category(G, guard_expression)
  		      end,
  		 Node1 = rewrite(Node, refac_syntax:clause(P1, G1, B1)),
  		 {refac_syntax:add_ann({category, clause}, Node1), true};
  	     match_expr ->
  		 P = refac_syntax:match_expr_pattern(Node),
  		 B = refac_syntax:match_expr_body(Node),
  		 P1 = add_category(P, pattern),
  		 B1 = add_category(B, C),
  		 Node1 = rewrite(Node, refac_syntax:match_expr(P1, B1)),
  		 {refac_syntax:add_ann({category, C}, Node1), true};
  	     operator -> {refac_syntax:add_ann({category, operator}, Node), true}; %% added to fix bug 13/09/2008.
  	     application ->
  		 Op = refac_syntax:application_operator(Node),
  		 Args = refac_syntax:application_arguments(Node),
  		 Op1 = add_category(Op, application_op),
  		 Args1 = add_category(Args, C),
  		 Node1 = rewrite(Node, refac_syntax:application(Op1, Args1)),
  		 {refac_syntax:add_ann({category, C}, Node1), true};
  	     arity_qualifier ->
  		 Fun = add_category(refac_syntax:arity_qualifier_body(Node), arity_qualifier),
  		 A = add_category(refac_syntax:arity_qualifier_argument(Node), arity_qualifier),
  		 Node1 = refac_syntax:arity_qualifier(Fun, A),
  		 {refac_syntax:add_ann({category, C}, Node1), true};
  	     macro ->
  		 Name = refac_syntax:macro_name(Node),
  		 Args = refac_syntax:macro_arguments(Node),
  		 Name1 = add_category(Name, macro_name),
  		 Args1 = case Args of
  			   none -> none;
  			   _ -> add_category(Args, expression) %% should 'expression' be 'macro_args'?
  			 end,
  		 Node1 = rewrite(Node, refac_syntax:macro(Name1, Args1)),
  		 {refac_syntax:add_ann({category, C}, Node1), true};
  	     record_access ->
  		 Argument = refac_syntax:record_access_argument(Node),
  		 Type = refac_syntax:record_access_type(Node),
  		 Field = refac_syntax:record_access_field(Node),
  		 Argument1 = add_category(Argument, C),
  		 Type1 = case Type of
  			   none -> none;
  			   _ -> add_category(Type, record_type)
  			 end,
  		 Field1 = add_category(Field, record_field),
  		 Node1 = rewrite(Node, refac_syntax:record_access(Argument1, Type1, Field1)),
  		 {refac_syntax:add_ann({category, C}, Node1), true};
  	     record_expr ->
  		 Argument = refac_syntax:record_expr_argument(Node),
  		 Type = refac_syntax:record_expr_type(Node),
  		 Fields = refac_syntax:record_expr_fields(Node),
  		 Argument1 = case Argument of
  			       none -> none;
  			       _ -> add_category(Argument, C)
  			     end,
  		 Type1 = add_category(Type, record_type),
  		 Fields1 = add_category(Fields, record_field),
  		 Node1 = rewrite(Node, refac_syntax:record_expr(Argument1, Type1, Fields1)),
  		 {refac_syntax:add_ann({category, C}, Node1), true};
  	     record_index_expr ->
  		 Type = refac_syntax:record_index_expr_type(Node),
  		 Field = refac_syntax:record_index_expr_field(Node),
  		 Type1 = add_category(Type, record_type),
  		 Field1 = add_category(Field, C),
  		 Node1 = rewrite(Node, refac_syntax:record_index_expr(Type1, Field1)),
  		 {refac_syntax:add_ann({category, C}, Node1), true};
  	     record_field ->
  		 Name = refac_syntax:record_field_name(Node),
  		 Name1 = add_category(Name, record_field),
  		 Value = refac_syntax:record_field_value(Node),
  		 Value1 = case Value of
  			    none -> none;
  			    _ -> add_category(Value, expression)
  			  end,
  		 Node1 = rewrite(Node, refac_syntax:record_field(Name1, Value1)),
  		 {refac_syntax:add_ann({category, record_field}, Node1), true};
  	     generator ->
  		 P = refac_syntax:generator_pattern(Node),
  		 B = refac_syntax:generator_body(Node),
  		 P1 = add_category(P, pattern),
  		 B1 = add_category(B, expression),
  		 Node1 = rewrite(Node, refac_syntax:generator(P1, B1)),
  		 {refac_syntax:add_ann({category, generator}, Node1), true};
  	     attribute ->
  		 case refac_syntax:atom_value(refac_syntax:attribute_name(Node)) of
  		   define ->
  		       Name = refac_syntax:attribute_name(Node),
  		       Args = refac_syntax:attribute_arguments(Node),
  		       MacroHead = refac_misc:ghead("Refac_util:do_add_category:MacroHead", Args),
  		       MacroBody = tl(Args),
  		       MacroHead1 = case refac_syntax:type(MacroHead) of
  				      application ->
  					  Operator = add_category(refac_syntax:application_operator(MacroHead), macro_name),
  					  Arguments = add_category(refac_syntax:application_arguments(MacroHead), attribute),
  					  rewrite(MacroHead, refac_syntax:application(Operator, Arguments));
  				      _ -> add_category(MacroHead, macro_name)
  				    end,
  		       MacroBody1 = add_category(MacroBody, attribute),
  		       Node1 = rewrite(Node, refac_syntax:attribute(Name, [MacroHead1| MacroBody1])),
  		       {refac_syntax:add_ann({category, attribute}, Node1), true};
  		   _ -> {refac_syntax:add_ann({category, C}, Node), false}
  		 end;
  	     %% TO ADD: other cases such as fields. Refer to the Erlang Specification.
  	     binary_field -> {refac_syntax:add_ann({category, binary_field}, Node), false};
  	     size_qualifier -> {refac_syntax:add_ann({category, size_qualifier}, Node), false};
  	     _ -> {refac_syntax:add_ann({category, C}, Node), false}
  	   end
      end.
  
  
  
  rewrite(Tree, Tree1) ->
      refac_syntax:copy_attrs(Tree, Tree1).
  
negative.code: 	     {functions, F(functions)}, {rules, F(rules)}],
      [{A,V}||{A, V}<-NewInfo, V=/=[]].
  	     
  
  annotate_bindings(FName, AST, Info, Ms, TabWidth) ->
      Toks = tokenize(FName, true, TabWidth),
      AnnAST0 = refac_syntax_lib:annotate_bindings(add_token_and_ranges(AST, Toks), ordsets:new(), Ms),
      refac_annotate_ast:add_fun_define_locations(AnnAST0, Info).
  
    
  %% Attach tokens to each form in the AST, and also add 
  %% range information to each node in the AST.
  %%-spec add_tokens(syntaxTree(), [token()]) -> syntaxTree(). 		 
  add_token_and_ranges(SyntaxTree, Toks) ->
      Fs = refac_syntax:form_list_elements(SyntaxTree),
      NewFs = do_add_token_and_ranges(Toks, Fs),
      SyntaxTree1= rewrite(SyntaxTree, refac_syntax:form_list(NewFs)),
      add_range_to_body(SyntaxTree1, NewFs, "", "").
  
  %% do it backwards starting from the last form. 
  %% all the white spaces after a form belong to the next form if
  %% there is one. 
  
  do_add_token_and_ranges(Toks, Fs) ->
      do_add_token_and_ranges(lists:reverse(Toks), lists:reverse(Fs), []).
  
  do_add_token_and_ranges(_, [], NewFs) ->
      NewFs;
  do_add_token_and_ranges(Toks, _Forms=[F| Fs], NewFs) ->
      {FormToks0, RemToks} = get_form_tokens(Toks, F, Fs), 
      FormToks = lists:reverse(FormToks0),
      F1 = refac_syntax:add_ann({toks, FormToks}, F),
      F2 = add_category(add_range(F1, FormToks)),
      do_add_token_and_ranges(RemToks, Fs, [F2| NewFs]).
  
  get_form_tokens(Toks, F, Fs) ->
      case refac_syntax:type(F) of
  	comment ->
  	    get_comment_form_toks(Toks, F, Fs);
  	_ ->
  	    get_non_comment_form_toks(Toks, F, Fs) 
      end.
  
  %% stand-alone comments.
  get_comment_form_toks(Toks, _F, Fs) when Fs==[] ->
      {Toks,[]};
  get_comment_form_toks(Toks, F, _Fs) ->
      StartPos =start_pos(F),
      {Ts1,Ts2} = lists:splitwith(
  		  fun(T) ->
  			  token_loc(T)>=StartPos 
  		  end, Toks),
      {Ts21, Ts22} = lists:splitwith(fun(T) ->
  					   is_whitespace(T)
  				   end, Ts2),
      {Ts1++Ts21, Ts22}.
   
  get_non_comment_form_toks(Toks, _F, Fs) when Fs==[] ->
      {Toks, []};
  get_non_comment_form_toks(Toks, F, _Fs) ->
      StartPos = start_pos(F),
      {Ts1, Ts2} = lists:splitwith(
  		   fun(T) ->
  			   token_loc(T)>=StartPos
  		   end, Toks),
      {Ts21, Ts22} = lists:splitwith(
  		     fun(T) ->
  			     element(1, T) /=dot andalso
  				 element(1,T)/=comment
  		     end, Ts2),
      {Ts1++Ts21, Ts22}.
  
  start_pos(F) ->
      case refac_syntax:type(F) of 
  	error_marker ->
  	    case F of
  		{error, {_, {{Line, Col}, {_Line1, _Col1}}}} ->
  		    {Line, Col};
  		_ ->
  		    refac_syntax:get_pos(F)
  	    end;
  	_ ->
  	    case refac_syntax:get_precomments(F) of
  		[] ->
  		    refac_syntax:get_pos(F);
  		[Com| _Tl] ->
  		    refac_syntax:get_pos(Com)
  	    end
      end.
  
  %%-spec add_range(syntaxTree(), [token()]) -> syntaxTree(). 
  add_range(AST, Toks) ->
  