text: import time
  start_time = time.time()
  
  import numpy as np
  import pandas as pd
  from sklearn.ensemble import RandomForestRegressor
  #from sklearn import pipeline, model_selection
  from sklearn import pipeline, grid_search
  #from sklearn.feature_extraction import DictVectorizer
  from sklearn.base import BaseEstimator, TransformerMixin
  from sklearn.pipeline import FeatureUnion
  from sklearn.decomposition import TruncatedSVD
  #from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer
  from sklearn.feature_extraction.text import TfidfVectorizer
  from sklearn.metrics import mean_squared_error, make_scorer
  #from nltk.metrics import edit_distance
  from nltk.stem.porter import *
  stemmer = PorterStemmer()
  #from nltk.stem.snowball import SnowballStemmer #0.003 improvement but takes twice as long as PorterStemmer
  #stemmer = SnowballStemmer('english')
  import re
  #import enchant
  import random
  random.seed(2015)
  
  df_train = pd.read_csv('../input/train.csv', encoding="ISO-8859-1")
  df_test = pd.read_csv('../input/test.csv', encoding="ISO-8859-1")
  df_pro_desc = pd.read_csv('../input/product_descriptions.csv')
  df_attr = pd.read_csv('../input/attributes.csv')
  df_brand = df_attr[df_attr.name == "MFG Brand Name"][["product_uid", "value"]].rename(columns={"value": "brand"})
  num_train = df_train.shape[0]
  df_all = pd.concat((df_train, df_test), axis=0, ignore_index=True)
  df_all = pd.merge(df_all, df_pro_desc, how='left', on='product_uid')
  df_all = pd.merge(df_all, df_brand, how='left', on='product_uid')
  
  def str_stem(s): 
      if isinstance(s, str):
          s = s.lower()
          s = re.sub(r"(\w)\.([A-Z])", r"\1 \2", s) ##'desgruda' palavras que estão juntas
  
          s = re.sub(r"([0-9]+)( *)(inches|inch|in|')\.?", r"\1in. ", s)
      
          s = re.sub(r"([0-9]+)( *)(foot|feet|ft|'')\.?", r"\1ft. ", s)
      
          s = re.sub(r"([0-9]+)( *)(pounds|pound|lbs|lb)\.?", r"\1lb. ", s)
      
          s = s.replace(" x "," xby ")
          s = s.replace("*"," xby ")
          s = s.replace(" by "," xby")
          s = s.replace("x0"," xby 0")
          s = s.replace("x1"," xby 1")
          s = s.replace("x2"," xby 2")
          s = s.replace("x3"," xby 3")
          s = s.replace("x4"," xby 4")
          s = s.replace("x5"," xby 5")
          s = s.replace("x6"," xby 6")
          s = s.replace("x7"," xby 7")
          s = s.replace("x8"," xby 8")
          s = s.replace("x9"," xby 9")
          s = s.replace("0x","0 xby ")
          s = s.replace("1x","1 xby ")
          s = s.replace("2x","2 xby ")
          s = s.replace("3x","3 xby ")
          s = s.replace("4x","4 xby ")
          s = s.replace("5x","5 xby ")
          s = s.replace("6x","6 xby ")
          s = s.replace("7x","7 xby ")
          s = s.replace("8x","8 xby ")
          s = s.replace("9x","9 xby ")
          
          s = re.sub(r"([0-9]+)( *)(square|sq) ?\.?(feet|foot|ft)\.?", r"\1sq.ft. ", s)
      
          s = re.sub(r"([0-9]+)( *)(gallons|gallon|gal)\.?", r"\1gal. ", s)
          
          s = re.sub(r"([0-9]+)( *)(ounces|ounce|oz)\.?", r"\1oz. ", s)
      
          s = re.sub(r"([0-9]+)( *)(centimeters|cm)\.?", r"\1cm. ", s)
      
          s = re.sub(r"([0-9]+)( *)(milimeters|mm)\.?", r"\1mm. ", s)
          
          s = re.sub(r"([0-9]+)( *)(degrees|degree)\.?", r"\1deg. ", s)
      
          s = re.sub(r"([0-9]+)( *)(volts|volt)\.?", r"\1volt. ", s)
          
          s = re.sub(r"([0-9]+)( *)(watts|watt)\.?", r"\1watt. ", s)
      
          s = re.sub(r"([0-9]+)( *)(amperes|ampere|amps|amp)\.?", r"\1amp. ", s)
          
          
          
          s = s.replace("whirpool","whirlpool")
          s = s.replace("whirlpoolga", "whirlpool")
          s = s.replace("whirlpoolstainless","whirlpool stainless")
  
          s = s.replace("  "," ")
          #s = (" ").join([stemmer.stem(z) for z in s.lower().split(" ")])
          s = (" ").join([stemmer.stem(z) for z in s.split(" ")])
          return s.lower()
      else:
          return "null"
  
  def str_common_word(str1, str2):
      words, cnt = str1.split(), 0
      for word in words:
          if str2.find(word)>=0:
              cnt+=1
      return cnt
  
  def str_whole_word(str1, str2, i_):
      cnt = 0
      while i_ < len(str2):
          i_ = str2.find(str1, i_)
          if i_ == -1:
              return cnt
          else:
              cnt += 1
              i_ += len(str1)
      return cnt
  
  def fmean_squared_error(ground_truth, predictions):
      fmean_squared_error_ = mean_squared_error(ground_truth, predictions)**0.5
      return fmean_squared_error_
  
  RMSE  = make_scorer(fmean_squared_error, greater_is_better=False)
  
  class cust_regression_vals(BaseEstimator, TransformerMixin):
      def fit(self, x, y=None):
          return self
      def transform(self, hd_searches):
          d_col_drops=['id','relevance','search_term','product_title','product_description','product_info','attr','brand']
          hd_searches = hd_searches.drop(d_col_drops,axis=1).values
          return hd_searches
  
  class cust_txt_col(BaseEstimator, TransformerMixin):
      def __init__(self, key):
          self.key = key
      def fit(self, x, y=None):
          return self
      def transform(self, data_dict):
          return data_dict[self.key].apply(str)
  
  def fmean_squared_error(ground_truth, predictions):
      fmean_squared_error_ = mean_squared_error(ground_truth, predictions)**0.5
      return fmean_squared_error_
  
  RMSE  = make_scorer(fmean_squared_error, greater_is_better=False)
  
  #comment out the lines below use df_all.csv for further grid search testing
  #if adding features consider any drops on the 'cust_regression_vals' class
  df_all['search_term'] = df_all['search_term'].map(lambda x:str_stem(x))
  df_all['product_title'] = df_all['product_title'].map(lambda x:str_stem(x))
  df_all['product_description'] = df_all['product_description'].map(lambda x:str_stem(x))
  df_all['brand'] = df_all['brand'].map(lambda x:str_stem(x))
  df_all['len_of_query'] = df_all['search_term'].map(lambda x:len(x.split())).astype(np.int64)
  df_all['len_of_title'] = df_all['product_title'].map(lambda x:len(x.split())).astype(np.int64)
  df_all['len_of_description'] = df_all['product_description'].map(lambda x:len(x.split())).astype(np.int64)
  df_all['len_of_brand'] = df_all['brand'].map(lambda x:len(x.split())).astype(np.int64)
  df_all['product_info'] = df_all['search_term']+"\t"+df_all['product_title'] +"\t"+df_all['product_description']
  df_all['query_in_title'] = df_all['product_info'].map(lambda x:str_whole_word(x.split('\t')[0],x.split('\t')[1],0))
  df_all['query_in_description'] = df_all['product_info'].map(lambda x:str_whole_word(x.split('\t')[0],x.split('\t')[2],0))
  df_all['word_in_title'] = df_all['product_info'].map(lambda x:str_common_word(x.split('\t')[0],x.split('\t')[1]))
  df_all['word_in_description'] = df_all['product_info'].map(lambda x:str_common_word(x.split('\t')[0],x.split('\t')[2]))
  df_all['ratio_title'] = df_all['word_in_title']/df_all['len_of_query']
  df_all['ratio_description'] = df_all['word_in_description']/df_all['len_of_query']
  df_all['attr'] = df_all['search_term']+"\t"+df_all['brand']
  df_all['word_in_brand'] = df_all['attr'].map(lambda x:str_common_word(x.split('\t')[0],x.split('\t')[1]))
  df_all['ratio_brand'] = df_all['word_in_brand']/df_all['len_of_brand']
  df_brand = pd.unique(df_all.brand.ravel())
  d={}
  i = 1
  for s in df_brand:
      d[s]=i
      i+=1
  df_all['brand_feature'] = df_all['brand'].map(lambda x:d[x])
  df_all['search_term_feature'] = df_all['search_term'].map(lambda x:len(x))
  #df_all.to_csv('df_all.csv')
  #df_all = pd.read_csv('df_all.csv', encoding="ISO-8859-1", index_col=0)
  df_train = df_all.iloc[:num_train]
  df_test = df_all.iloc[num_train:]
  id_test = df_test['id']
  y_train = df_train['relevance'].values
  X_train =df_train[:]
  X_test = df_test[:]
  print("--- Features Set: %s minutes ---" % round(((time.time() - start_time)/60),2))
  
  rfr = RandomForestRegressor(n_estimators = 700,max_features=10,max_depth=20,n_jobs = -1, random_state = 2016, verbose = 1)
  tfidf = TfidfVectorizer(ngram_range=(1, 1), stop_words='english')
  tsvd = TruncatedSVD(n_components=11, random_state = 2016)
  model = pipeline.Pipeline([
          ('union', FeatureUnion(
                      transformer_list = [
                          ('cst',  cust_regression_vals()),  
                          ('txt1', pipeline.Pipeline([('s1', cust_txt_col(key='search_term')), ('tfidf1', tfidf), ('tsvd1', tsvd)])),
                          ('txt2', pipeline.Pipeline([('s2', cust_txt_col(key='product_title')), ('tfidf2', tfidf), ('tsvd2', tsvd)])),
                          ('txt3', pipeline.Pipeline([('s3', cust_txt_col(key='product_description')), ('tfidf3', tfidf), ('tsvd3', tsvd)])),
                          ('txt4', pipeline.Pipeline([('s4', cust_txt_col(key='brand')), ('tfidf4', tfidf), ('tsvd4', tsvd)]))
                          ],
                      transformer_weights = {
                          'cst': 1.0,
                          'txt1': 0.5,
                          'txt2': 0.25,
                          'txt3': 0.0,
                          'txt4': 0.5
                          },
                  n_jobs = -1
                  )), 
          ('rfr', rfr)])
  model.fit(X_train, y_train)
  y_pred = model.predict(X_test)
  pd.DataFrame({"id": id_test, "relevance": y_pred}).to_csv('submission.csv',index=False)
  print("--- Training & Testing: %s minutes ---" % round(((time.time() - start_time)/60),2))