title: When to cache a DataFrame?
body: <p>My question is - when should I do <code>dataframe.cache()</code> and when it's useful?</p>
  <p>Also, in my code should I cache the dataframes in the commented lines?</p>
  <p>Note: My dataframes are loaded from a Redshift DB.</p>
  <p>Many thanks</p>
  <p>Here my code:</p>
  <pre><code>def sub_tax_transfer_pricing_eur_aux(manager, dataframe, seq_recs, seq_reservas, df_aux):
      df_vta = manager.get_dataframe(tables['dwc_oth_v_re_v_impuesto_sap_vta'])
      df_cpa = manager.get_dataframe(tables['dwc_oth_v_re_v_impuesto_sap_cpa'])
      
      dataframe = dataframe.filter(dataframe.seq_rec.isin(seq_recs)) \
          .filter(dataframe.seq_reserva.isin(seq_reservas))
  
      ##################################################
      #SHOULD I CACHE HERE df_vta, df_cpa and dataframe
      ##################################################
  
      dataframe = dataframe.join(df_vta, [dataframe.ind_tipo_imp_vta_fac == df_vta.ind_tipo_imp_vta,
                                          dataframe.cod_impuesto_vta_fac == df_vta.cod_impuesto_vta,
                                          dataframe.cod_clasif_vta_fac == df_vta.cod_clasif_vta,
                                          dataframe.cod_esquema_vta_fac == df_vta.cod_esquema_vta,
                                          dataframe.cod_empresa_vta_fac == df_vta.cod_emp_atlas_vta,
                                          ]).drop(&quot;ind_tipo_imp_vta&quot;, &quot;cod_impuesto_vta&quot;, &quot;cod_clasif_vta&quot;,
                                                  &quot;cod_esquema_vta&quot;, &quot;cod_emp_atlas_vta&quot;) \
          .join(df_cpa, [dataframe.ind_tipo_imp_vta_fac == df_cpa.ind_tipo_imp_cpa,
                         dataframe.cod_impuesto_vta_fac == df_cpa.cod_impuesto_cpa,
                         dataframe.cod_clasif_vta_fac == df_cpa.cod_clasif_cpa,
                         dataframe.cod_esquema_vta_fac == df_cpa.cod_esquema_cpa,
                         dataframe.cod_empresa_vta_fac == df_cpa.cod_emp_atlas_cpa,
                         ]).drop(&quot;ind_tipo_imp_cpa&quot;, &quot;cod_impuesto_cpa&quot;, &quot;cod_clasif_cpa&quot;,
                                 &quot;cod_esquema_cpa&quot;, &quot;cod_emp_atlas_cpa&quot;) \
          .select(&quot;seq_rec&quot;, &quot;seq_reserva&quot;, &quot;ind_tipo_regimen_fac&quot;, &quot;imp_margen_canal&quot;, &quot;ind_tipo_regimen_con&quot;,
                  &quot;imp_coste&quot;, &quot;imp_margen_canco&quot;, &quot;imp_venta&quot;, &quot;pct_impuesto_vta&quot;, &quot;pct_impuesto_cpa&quot;)
  
      ######################################         
      #SHOULD I CACHE HERE dataframe AGAIN ?
      ######################################
  
      dataframe = dataframe.withColumn(&quot;amount1&quot;,
                                       func.when(dataframe.ind_tipo_regimen_fac == 'E',
                                                 dataframe.imp_margen_canal * (
                                                     1 - (1 / (1 + (dataframe.pct_impuesto_vta
                                                                    / 100)))))
                                       .otherwise(dataframe.imp_venta * (
                                           1 - (1 / (1 + (dataframe.pct_impuesto_vta / 100)))) - (
                                                      dataframe.imp_venta - dataframe.imp_margen_canal) * (
                                                      1 - (1 / (1 + (dataframe.pct_impuesto_cpa / 100))))))
  
      dataframe = dataframe.withColumn(&quot;amount2&quot;,
                                       func.when(dataframe.ind_tipo_regimen_con == 'E',
                                                 dataframe.imp_margen_canco * (
                                                     1 - (1 / (1 + (dataframe.pct_impuesto_vta
                                                                    / 100)))))
                                       .otherwise((dataframe.imp_coste + dataframe.imp_margen_canco) * (
                                           1 - (1 / (1 + (dataframe.pct_impuesto_vta / 100)))) - (
                                                      dataframe.imp_coste) * (
                                                      1 - (1 / (1 + (dataframe.pct_impuesto_cpa / 100))))))
  
      dataframe = dataframe.na.fill({'amount1': 0})
      dataframe = dataframe.na.fill({'amount2': 0})
  
      dataframe = dataframe.join(df_aux, [dataframe.seq_rec == df_aux.operative_incoming,
                                          dataframe.seq_reserva == df_aux.booking_id])
  
      dataframe = dataframe.withColumn(&quot;impuesto_canco1&quot;, udf_currency_exchange(dataframe.booking_currency,
                                                                               func.lit(EUR),
                                                                               dataframe.creation_date,
                                                                               dataframe.amount1))
  
      dataframe = dataframe.withColumn(&quot;impuesto_canco2&quot;, udf_currency_exchange(dataframe.booking_currency,
                                                                               func.lit(EUR),
                                                                               dataframe.creation_date,
                                                                               dataframe.amount2))
  
      dataframe = dataframe.withColumn(&quot;impuesto_canco&quot;, dataframe.impuesto_canco1 + dataframe.impuesto_canco2)
  
      dataframe = dataframe.na.fill({'impuesto_canco': 0})
  
      dataframe = dataframe.select(&quot;operative_incoming&quot;, &quot;booking_id&quot;, &quot;impuesto_canco&quot;)
      ######################################         
      #SHOULD I CACHE HERE dataframe AGAIN ?
      ######################################
      dataframe = dataframe.groupBy(&quot;operative_incoming&quot;, &quot;booking_id&quot;).agg({'impuesto_canco': 'sum'}). \
          withColumnRenamed(&quot;SUM(impuesto_canco)&quot;, &quot;impuesto_canco&quot;)
  
      return dataframe
  </code></pre>