title: Merging large datasets
body: <p>Im trying to categorize one dataframe based on information in another dataframe. In df1 I have information on the measurement type (e.g. if a jar contained wet or dry soil and whether or not the treatment was &quot;None&quot; or &quot;ul5&quot;) at a given time. In df2 I have information on what a measured value X was at a given time. I need to know the measurement type for every measured value of X.</p>
  <p>@Ronak Shah came up with this great solution below, however since the datasets are large I get this Error: cannot allocate vector of size 56.2 Gb</p>
  <pre><code>library(dplyr)
  
  tidyr::crossing(df1 %&gt;%rename(Timestamp1 = Timestamp), 
                  df2 %&gt;% rename(Timestamp2 = Timestamp)) %&gt;%
    mutate(diff = as.numeric(Timestamp2 - Timestamp1)) %&gt;%
    filter(diff &gt; 0) %&gt;% 
    arrange(Jar, Timestamp2, diff) %&gt;%
    group_by(Timestamp2) %&gt;%
    slice(1L) %&gt;%
    ungroup %&gt;%
    arrange(Timestamp2) %&gt;%
    select(-diff) 
  </code></pre>
  <p>Any ideas as how to merge the large dataset? I have an ThinkPad intel Corei7 8th Gen, so my computer is not super slow.</p>
  <p>Here's df1:</p>
  <pre><code>df1 &lt;- structure(list(Jar = c(&quot;Soil_dry&quot;, &quot;Soil_dry&quot;, &quot;soil_wet&quot;, &quot;soil_wet&quot;, 
  &quot;Soil_dry&quot;, &quot;Soil_dry&quot;, &quot;soil_wet&quot;), Treatment = c(&quot;None&quot;, &quot;None&quot;, 
  &quot;None&quot;, &quot;None&quot;, &quot;ul5&quot;, &quot;ul5&quot;, &quot;ul5&quot;), Timestamp = structure(c(1608129063, 
  1608129122, 1608129126, 1608129136, 1608129189, 1608129242, 1608129252
  ), class = c(&quot;POSIXct&quot;, &quot;POSIXt&quot;), tzone = &quot;UTC&quot;)), class = c(&quot;spec_tbl_df&quot;, 
  &quot;tbl_df&quot;, &quot;tbl&quot;, &quot;data.frame&quot;), row.names = c(NA, -7L), spec = structure(list(
      cols = list(Jar = structure(list(), class = c(&quot;collector_character&quot;, 
      &quot;collector&quot;)), Treatment = structure(list(), class = c(&quot;collector_character&quot;, 
      &quot;collector&quot;)), Timestamp = structure(list(format = &quot;&quot;), class = c(&quot;collector_datetime&quot;, 
      &quot;collector&quot;))), default = structure(list(), class = c(&quot;collector_guess&quot;, 
      &quot;collector&quot;)), skip = 1), class = &quot;col_spec&quot;))
  </code></pre>
  <p>df2:</p>
  <pre><code>df2 &lt;- structure(list(X = c(5, 3, 34, 4, 65, 9, 7), Timestamp = structure(c(1608129064, 
  1608129122, 1608129125, 1608129133, 1608129188, 1608129240, 1608129243
  ), class = c(&quot;POSIXct&quot;, &quot;POSIXt&quot;), tzone = &quot;UTC&quot;)), class = c(&quot;spec_tbl_df&quot;, 
  &quot;tbl_df&quot;, &quot;tbl&quot;, &quot;data.frame&quot;), row.names = c(NA, -7L), spec = structure(list(
      cols = list(X = structure(list(), class = c(&quot;collector_double&quot;, 
      &quot;collector&quot;)), Timestamp = structure(list(format = &quot;&quot;), class = c(&quot;collector_datetime&quot;, 
      &quot;collector&quot;))), default = structure(list(), class = c(&quot;collector_guess&quot;, 
      &quot;collector&quot;)), skip = 1), class = &quot;col_spec&quot;))
  </code></pre>
  <p>Desired data:</p>
  <pre><code>desired_data &lt;- structure(list(X = c(5, 3, 34, 4, 65, 9, 7), Timestamp = structure(c(1608129064, 
  1608129122, 1608129125, 1608129133, 1608129188, 1608129240, 1608129243
  ), class = c(&quot;POSIXct&quot;, &quot;POSIXt&quot;), tzone = &quot;UTC&quot;), Jar = c(&quot;Soil_dry&quot;, 
  &quot;Soil_dry&quot;, &quot;Soil_dry&quot;, &quot;soil_wet&quot;, &quot;soil_wet&quot;, &quot;Soil_dry&quot;, &quot;Soil_dry&quot;
  ), Treatment = c(&quot;None&quot;, &quot;None&quot;, &quot;None&quot;, &quot;None&quot;, &quot;None&quot;, &quot;ul5&quot;, 
  &quot;ul5&quot;)), class = c(&quot;spec_tbl_df&quot;, &quot;tbl_df&quot;, &quot;tbl&quot;, &quot;data.frame&quot;
  ), row.names = c(NA, -7L), spec = structure(list(cols = list(
      X = structure(list(), class = c(&quot;collector_double&quot;, &quot;collector&quot;
      )), Timestamp = structure(list(format = &quot;&quot;), class = c(&quot;collector_datetime&quot;, 
      &quot;collector&quot;)), Jar = structure(list(), class = c(&quot;collector_character&quot;, 
      &quot;collector&quot;)), Treatment = structure(list(), class = c(&quot;collector_character&quot;, 
      &quot;collector&quot;))), default = structure(list(), class = c(&quot;collector_guess&quot;, 
  &quot;collector&quot;)), skip = 1), class = &quot;col_spec&quot;))
  </code></pre>