anchor.diff:  insert_to_ast_tab(Pid, {{M, F, A}, ExprASTs}) ->
       Pid ! {add, {{M, F, A}, ExprASTs}}.
   
  -get_clone_candidates(Pid, MinLen, MinToks, MinFreq, Dir) ->
  -    Pid ! {get_clone_candidates, self(), MinLen, MinToks, MinFreq, Dir},
  +get_clone_candidates(Pid, Thresholds, Dir) ->
  +    Pid ! {get_clone_candidates, self(), Thresholds, Dir},
       receive
         {Pid, Cs} ->
   	  Cs
  
positive.code: -define(ExpHashTab, list_to_atom(filename:join(?WRANGLER_DIR, "plt/exp_hash_tab"))).
  -define(ExpSeqFile, list_to_atom(filename:join(?WRANGLER_DIR, "plt/exp_seq_file"))).
  
  
  -spec(inc_sim_code_detection/7::(DirFileList::[filename()|dir()], MinLen::float(), MinToks::integer(),MinFreq::integer(), MinScore::float(), 
  			     SearchPaths::[dir()], TabWidth::integer()) -> {ok, string()}). 			     
  inc_sim_code_detection(DirFileList, MinLen, MinToks, MinFreq, SimiScore, SearchPaths, TabWidth) ->
      ?wrangler_io("\nCMD: ~p:sim_code_detection(~p,~p,~p,~p, ~p,~p,~p).\n",
  		 [?MODULE, DirFileList, MinLen, MinToks, MinFreq, SimiScore, SearchPaths, TabWidth]),
      Files = refac_util:expand_files(DirFileList, ".erl"),
      ASTTab =from_dets(ast_tab, ?ASTTab),
      VarTab= from_dets(var_tab, ?VarTab),
      FileHashTab=from_dets(file_hash_tab,?FileHashTab),
      ExpHashTab =from_dets(expr_hash_tab, ?ExpHashTab),
      ASTPid = start_ast_process(ASTTab),
      case Files of
        [] ->
  	  ?wrangler_io("Warning: No files found in the searchpaths specified.", []);
        _ ->
  	    _Time1 = now(),
  	    generalise_and_hash_ast(Files, FileHashTab, ASTPid, MinLen, VarTab, SearchPaths, TabWidth),
  	    ?wrangler_io("Generalise and has hash finished.\n",[]),
  	    Dir = filename:dirname(hd(Files)),
  	    Cs = get_clone_candidates(ASTPid, MinLen, MinToks, MinFreq, Dir),
  	    ?debug("\nInitial candiates finished\n", []),
  	    ?wrangler_io("\nNumber of initial clone candidates: ~p\n", [length(Cs)]),
  	    CloneCheckerPid = start_clone_check_process(),
  	    Cs2 = examine_clone_candidates(Cs, MinLen, MinFreq, SimiScore, ASTTab, VarTab, CloneCheckerPid, 1),
  	    stop_clone_check_process(CloneCheckerPid),
  	    _Time2 = now(),
  	    refac_code_search_utils:display_clone_result(Cs2, "Similar")
      end,
      stop_ast_process(ASTPid),
      to_dets(ASTTab, ?ASTTab),
      to_dets(VarTab, ?VarTab),
      to_dets(FileHashTab, ?FileHashTab),
      to_dets(ExpHashTab, ?ExpHashTab).
      
   
  %% Serialise, in breath-first order, and generalise each expression in the AST,
  
negative.code:     end.
   
  %%=============================================================================
  examine_clone_candidates([], _Thresholds, Tabs,_TmpASTTab, Pid, _Num) ->
      get_final_clone_classes(Pid, Tabs#tabs.ast_tab);
  examine_clone_candidates([C| Cs], Thresholds, Tabs, TmpASTTab, Pid, Num) ->
      output_progress_msg(Num),
      case examine_a_clone_candidate(C, Thresholds, Tabs, TmpASTTab) of
  	[] ->
  	  ok;
  	ClonesWithAU ->
  	    add_new_clones(Pid, ClonesWithAU)
      end,
      examine_clone_candidates(Cs, Thresholds, Tabs, TmpASTTab,Pid, Num + 1).
  
  examine_a_clone_candidate(_C={Ranges, {_Len, _Freq}}, Thresholds,Tabs, TmpASTTab) ->
      ASTTab=Tabs#tabs.ast_tab,
      Ranges1=lists:append(Ranges),
      lists:foreach(fun({Key,_, _}) ->
  			  ets:insert(TmpASTTab, ets:lookup(ASTTab, Key))
  		  end, Ranges1),
      Clones=examine_clone_class_members(Ranges, Thresholds,Tabs, TmpASTTab, []),
      ClonesWithAU = [{Rs, {Len, Freq}, get_anti_unifier(TmpASTTab, Info)}
       		    || {Rs, {Len, Freq}, Info} <- Clones],
      ClonesWIthAUInAbsoluteLocs=[get_clone_class_in_absolute_locs(Clone)||Clone<-ClonesWithAU],
      ets:delete_all_objects(TmpASTTab),
      ClonesWIthAUInAbsoluteLocs.
     
  
  examine_clone_class_members(Rs, Thresholds, _, _, Acc) 
    when length(Rs)< Thresholds#threshold.min_freq ->
      remove_sub_clones(Acc);
  
  examine_clone_class_members(Ranges, Thresholds,Tabs, TmpASTTab, Acc) ->
      %%refac_io:format("Ranges:\n~p\n", [Ranges]),
      [Range1|Rs]=Ranges,
      Exprs1 = get_expr_list(Range1, TmpASTTab),
      Res = [begin 
  	       Exprs2= get_expr_list(Range2, TmpASTTab),
  	       do_anti_unification({Range1, Exprs1}, {Range2, Exprs2})
  	   end|| Range2<-Rs,
  		 Range2/=Range1],
      InitialLength = length(Range1),
      %% refac_io:format("Freq:\n~p\n", [length(Ranges)]),
      %% refac_io:format("InitaialLength:\n~p\n",[InitialLength]),
      Clones = process_au_result(Res, Thresholds, Tabs, TmpASTTab),
      MaxCloneLength= case Clones ==[] of 
  			true -> 0;
  			_-> element(1, element(2, hd(Clones)))
  		    end,
      %% refac_io:format("MaxCloneLength:\n~p\n", [MaxCloneLength]),
      case MaxCloneLength /= InitialLength of
  	true ->
  	    examine_clone_class_members(Rs, Thresholds,Tabs, TmpASTTab, Clones ++ Acc);
  	false ->
  	    Rs1 = element(1, hd(Clones)),
  	    RemainedRanges = Ranges -- Rs1,
  	    examine_clone_class_members(RemainedRanges, Thresholds,Tabs, TmpASTTab, Clones ++ Acc)
  
      end.
  
  process_au_result(AURes, Thresholds, Tabs, TmpASTTab) ->
      %% refac_io:format("AURES:\n~p\n", [length(AURes)]),
      Res = [process_one_au_result(OneAURes, Thresholds, Tabs, TmpASTTab)
  	   || OneAURes <- AURes],
      %% refac_io:format("RES:\n~p\n",[Res]),
      ClonePairs = lists:append(Res),
      get_clone_classes(ClonePairs, Thresholds, Tabs, TmpASTTab).
  
  process_one_au_result(OneAURes, Thresholds, _Tabs, TmpASTTab) ->
      SubAULists=group_au_result(OneAURes),
      %% refac_io:format("SubAUList:\n~p\n", [length(SubAULists)]),
      ClonePairs =lists:append([get_clone_pairs(SubAUList, Thresholds, TmpASTTab)||SubAUList<-SubAULists]),
      ClonePairs1 =[lists:unzip3(CP)||CP<-ClonePairs],
      remove_sub_clone_pairs(ClonePairs1).
  
  group_au_result([])->
      [];
  group_au_result(AURes) ->
      {AUResList1,AUResList2} =
  	lists:splitwith(fun({_,_, S}) ->S/=none end, AURes),
      case AUResList2 of
  	[] ->
  	    [AUResList1];
  	[_|T] ->
  	    case AUResList1/=[] of
  		true ->
  		    [AUResList1]++group_au_result(T);
  		false ->
  		    group_au_result(T)
  	    end
      end.
  
  get_clone_pairs(AURes, Thresholds, TmpASTTab) ->
      ClonePairs=get_clone_pairs(AURes, Thresholds, TmpASTTab, {[],[]},[]),
     %% refac_io:format("ClonePairs:\n~p\n", [ClonePairs]),
      ClonePairs.
  get_clone_pairs([], Thresholds, TmpASTTab, {_, ClonePairAcc}, Acc) ->
      case length(ClonePairAcc) < Thresholds#threshold.min_len of
  	true -> Acc;
  	_ -> 
  	    ClonePairAcc1 = decompose_clone_pair_by_simi_score(
  			      lists:reverse(ClonePairAcc), Thresholds, TmpASTTab),
  	    ClonePairAcc1++Acc
      end;  
  get_clone_pairs([CurPair={_, _,SubSt}|AURes], Thresholds, TmpASTTab, 
  		{VarSubAcc, ClonePairAcc}, Acc) ->
      VarSub = get_var_subst(SubSt),
      case var_sub_conflicts(VarSub,VarSubAcc) of
  	true ->
  	    case length(ClonePairAcc)>=Thresholds#threshold.min_len of
  		true ->
  		    ClonePairAcc1 = decompose_clone_pair_by_simi_score(
  				      lists:reverse(ClonePairAcc), Thresholds, TmpASTTab),
  		    NewAcc = ClonePairAcc1++Acc,			     
  		    get_clone_pairs(AURes, Thresholds, TmpASTTab, {[], []},NewAcc);
  		false ->
  		    get_clone_pairs(AURes, Thresholds, TmpASTTab, {[], []},Acc)
  	    end;
  	false ->
  	    get_clone_pairs(AURes, Thresholds, TmpASTTab, 
  			    {VarSub++VarSubAcc, [CurPair]++ClonePairAcc}, Acc)
      end.
  
  
  decompose_clone_pair_by_simi_score(ClonePair, Thresholds, TmpASTTab) ->
      {Range1, Range2, Subst} = lists:unzip3(ClonePair),
      Exprs1 = get_expr_list(Range1, TmpASTTab),
      Exprs2 = get_expr_list(Range2, TmpASTTab),
      {SubExprs1, SubExprs2} = lists:unzip(lists:append(Subst)),
      Score1 = simi_score(Exprs1, SubExprs1),
      case Score1 >= Thresholds#threshold.simi_score andalso
  	   simi_score(Exprs2, SubExprs2) >= Thresholds#threshold.simi_score
  	of
        true ->
  	  [ClonePair];
        false ->
  	  RangeExprPairs1 = lists:zip(Range1, Exprs1),
  	  RangeExprPairs2 = lists:zip(Range2, Exprs2),
  	  ClonePairsWithExpr = lists:zip3(RangeExprPairs1, RangeExprPairs2, Subst),
  	  %% refac_io:format("ClonePairswithExpr:\n~p\n",[ClonePairsWithExpr]),
  	  ClonePairWithSimiScore = [{{ExprKey1, Expr1}, {ExprKey2, Expr2}, Sub,
  				     simi_score(Expr1, SubEs1), simi_score(Expr2, SubEs2)}
  				    || {{ExprKey1, Expr1}, {ExprKey2, Expr2}, Sub} <- ClonePairsWithExpr,
  				       {SubEs1, SubEs2} <- [lists:unzip(Sub)]],
  	  decompose_clone_pair_by_simi_score_1(ClonePairWithSimiScore, Thresholds, [])
      end.
  
  decompose_clone_pair_by_simi_score_1([], _, Acc)->
      Acc;
  decompose_clone_pair_by_simi_score_1(ClonePairWithSimiScore, Thresholds, Acc) ->
      ClonePairs1 = lists:dropwhile(fun({_, _, _, Score1, Score2}) ->
  					  Score1 < Thresholds#threshold.simi_score*0.9 orelse
  					      Score2 < Thresholds#threshold.simi_score*0.9
  				  end, ClonePairWithSimiScore),
      case length(ClonePairs1)>=Thresholds#threshold.min_len of 
  	false ->
  	   %% refac_io:format("\nDDDDDD\n"),
  	    Acc;
  	true ->
  	    {ClonePairs2, ClonePairs3} =
  		lists:splitwith(fun({_, _, _, Score1, Score2}) ->
  					 Score1 >= Thresholds#threshold.simi_score*0.9 andalso
  					     Score2 >= Thresholds#threshold.simi_score*0.9
  				end, ClonePairs1),
  	    case length(ClonePairs2)>=Thresholds#threshold.min_len of
  		true ->
  		    NewClonePairs2=[{ExprKey1, ExprKey2, Sub}||{{ExprKey1,_}, {ExprKey2,_}, Sub, _,_}<-ClonePairs2],
  		    case length(ClonePairs3)>=Thresholds#threshold.min_len of
  			true ->
  			    decompose_clone_pair_by_simi_score_1(ClonePairs3, Thresholds, [NewClonePairs2|Acc]);  
  			false ->
  			    [NewClonePairs2|Acc]
  		    end;
  		false ->
  		    case length(ClonePairs3)>=Thresholds#threshold.min_len  of
  			true ->
  			    decompose_clone_pair_by_simi_score_1(ClonePairs3, Thresholds, Acc);
  			false ->
  			    Acc
  		    end
  	    end
      end.
  	    
      
  
  
  
  simi_score(Expr, SubExprs) ->
      case no_of_nodes(Expr) of
        0 -> 0;
        ExprSize ->
  	    NonVarExprs = [E || E <- SubExprs, refac_syntax:type(E) =/= variable],
  	    NoOfNewVars = length(NonVarExprs),
  	    Res = 1 - (no_of_nodes(SubExprs) - length(SubExprs)
  		       + NoOfNewVars * (NoOfNewVars + 1) / 2) / ExprSize,
  	    %% Res =1 -((no_of_nodes(SubExprs)-length(SubExprs))/ExprSize),
  	    Res
      end.
     
  
  var_sub_conflicts(VarSubs, CurVarSubs) ->
      lists:any(fun({DefPos, E})->
  			 case lists:keysearch(DefPos, 1, CurVarSubs) of
  			     {value, {DefPos, E1}} ->
  				 E/=E1;
  			     false ->
  				 false
  			 end
  	      end, VarSubs).
  
  get_var_subst(SubSt) ->
      F=fun({E1, E2}) ->
  	      {value, {def, DefPos}} = lists:keysearch(def, 1, refac_syntax:get_ann(E1)),
  	      {DefPos, refac_prettypr:format(reset_attrs(E2))}
        end,	      
      [F({E1,E2})||{E1,E2}<-SubSt, 
      refac_syntax:type(E1)==variable, not is_macro_name(E1)].
  
  remove_sub_clone_pairs([]) ->[];
  remove_sub_clone_pairs(CPs) ->
      SortedCPs = lists:sort(fun({Rs1,_,_}, {Rs2, _, _}) ->
  					  length(Rs1)>length(Rs2)
  				  end, CPs),
      %% refac_io:format("SoredCPs:\n~p\n",[SortedCPs]),
      remove_sub_clone_pairs(SortedCPs, []).
  remove_sub_clone_pairs([], Acc) ->
      lists:reverse(Acc);
  remove_sub_clone_pairs([CP={Rs, _,_}|CPs], Acc) ->
      case lists:any(fun({Rs1, _,_}) ->
  			   Rs--Rs1==[] 
  		   end, Acc) of
  	true ->
  	    remove_sub_clone_pairs(CPs,Acc);
  	_ -> remove_sub_clone_pairs(CPs, [CP|Acc])
      end.
  		
  get_clone_classes(ClonePairs,Thresholds, Tabs, TmpASTTab) ->
      RangeGroups = lists:usort([Rs || {Rs, _, _} <- ClonePairs]),
      %%refac_io:format("ClonePairs:\n~p\n", [ClonePairs]),
      %% refac_io:format("RangeGroups:\n~p\n", [RangeGroups]),
      CloneClasses = lists:append([get_one_clone_class(Range, ClonePairs, Thresholds, Tabs, TmpASTTab) 
  				 || Range <- RangeGroups]),
      %%refac_io:format("CloneClasses:\n~p\n", [CloneClasses]),
      lists:keysort(2, CloneClasses).
   
  get_one_clone_class(Range, ClonePairs, Thresholds, Tabs, TmpASTTab) ->
      Res = lists:append([get_one_clone_class_1(Range, ClonePair, Tabs, TmpASTTab)
  			|| ClonePair <- ClonePairs]),
      Freq = length(Res) + 1,
      case Freq >= Thresholds#threshold.min_freq of
        true ->
  	  Exprs = get_expr_list(Range, TmpASTTab),
  	  [{{FName, FunName, Arity, _}, _, _}| _] = Range,
  	  VarTab = Tabs#tabs.var_tab,
  	  VarsToExport = get_vars_to_export(Exprs, {FName, FunName, Arity}, VarTab),
  	  {Ranges, ExportVars, SubSt} = lists:unzip3(Res),
  	  ExportVars1 = {element(1, lists:unzip(VarsToExport)), lists:usort(lists:append(ExportVars))},
  	  [{[Range| Ranges], {length(Range), length(Ranges) + 1}, {Range, SubSt, ExportVars1}}];
        _ ->
  	  []
      end.
  get_one_clone_class_1(Range, _ClonePair = {Range1, Range2, Subst}, Tabs, TmpASTTab) ->
      case Range -- Range1 == [] of
        true ->
  	    Len = length(Range),
  	    R = hd(Range),
  	    StartIndex=length(lists:takewhile(fun (R0) -> R0 /= R end, Range1))+1,
  	   %% refac_io:format("StartIndex:\n~p\n",[StartIndex]),
  	    SubRange2 = lists:sublist(Range2, StartIndex, Len),
  	    SubSubst = lists:append(lists:sublist(Subst, StartIndex, Len)),
  	    Exprs2 = get_expr_list(SubRange2, TmpASTTab),
  	    [{{FName, FunName, Arity, _}, _, _}| _] = SubRange2,
  	    VarTab = Tabs#tabs.var_tab,
  	    VarsToExport2 = get_vars_to_export(Exprs2, {FName, FunName, Arity}, VarTab),
  	    EVs = [E1 || {E1, E2} <- SubSubst, refac_syntax:type(E2) == variable,
  			 lists:member({refac_syntax:variable_name(E2), get_var_define_pos(E2)},
  				      VarsToExport2)],
  	    [{SubRange2, EVs, SubSubst}];
        	false ->
  	    []
      end.
  
  					 
      
  
  
  
  do_anti_unification({Range1,Exprs1}, {Range2, Exprs2}) ->
     %% refac_io:format("Data:\n~p\n",[{{Range1,Exprs1}, {Range2, Exprs2}}]),
      ZippedExprs1=lists:zip(Range1, Exprs1),
      ZippedExprs2=lists:zip(Range2, Exprs2),
      ZippedExprs=lists:zip(ZippedExprs1, ZippedExprs2),
      [begin
  	 {Index1, Index2,
  	  do_anti_unification_1(E1,E2)}
       end|| {{Index1,E1}, {Index2, E2}}<-ZippedExprs].
      
  do_anti_unification_1(E1, E2) ->
      SubSt=anti_unification:anti_unification(E1,E2),
      case SubSt of 
  	none -> none;
  	_ -> case subst_sanity_check(E1, SubSt) of
  		 true ->
  		     SubSt;
  		 false ->
  		     none
  	     end
      end.
  
  %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  %%                                                                  %%
  %%  Remove sub-clones                                               %%
  %%                                                                  %%
  %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
  remove_sub_clones(Cs) ->
      remove_sub_clones(lists:reverse(lists:keysort(2,Cs)),[]).
  remove_sub_clones([], Acc_Cs) ->
      lists:reverse(Acc_Cs);
  remove_sub_clones([C|Cs], Acc_Cs) ->
  