text: /*
   * Licensed to the Apache Software Foundation (ASF) under one
   * or more contributor license agreements.  See the NOTICE file
   * distributed with this work for additional information
   * regarding copyright ownership.  The ASF licenses this file
   * to you under the Apache License, Version 2.0 (the
   * "License"); you may not use this file except in compliance
   * with the License.  You may obtain a copy of the License at
   * 
   *   http://www.apache.org/licenses/LICENSE-2.0
   * 
   * Unless required by applicable law or agreed to in writing,
   * software distributed under the License is distributed on an
   * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
   * KIND, either express or implied.  See the License for the
   * specific language governing permissions and limitations
   * under the License.
   */
  
  
  #include "net/instaweb/rewriter/public/critical_images_finder.h"
  
  #include <map>
  
  #include "base/logging.h"
  #include "net/instaweb/http/public/request_context.h"
  #include "net/instaweb/rewriter/critical_images.pb.h"
  #include "net/instaweb/rewriter/public/critical_finder_support_util.h"
  #include "net/instaweb/rewriter/public/property_cache_util.h"
  #include "net/instaweb/rewriter/public/rewrite_driver.h"
  #include "net/instaweb/rewriter/public/rewrite_options.h"
  #include "net/instaweb/rewriter/public/server_context.h"
  #include "net/instaweb/rewriter/rendered_image.pb.h"
  #include "net/instaweb/util/public/fallback_property_page.h"
  #include "net/instaweb/util/public/property_cache.h"
  #include "pagespeed/kernel/base/json.h"
  #include "pagespeed/kernel/base/message_handler.h"
  #include "pagespeed/kernel/base/proto_util.h"
  #include "pagespeed/kernel/base/scoped_ptr.h"
  #include "pagespeed/kernel/base/statistics.h"
  #include "pagespeed/kernel/base/string_util.h"
  #include "pagespeed/kernel/http/google_url.h"
  #include "pagespeed/opt/logging/log_record.h"
  
  namespace net_instaweb {
  
  namespace {
  
  const char kRenderedImageJsonWidthKey[] = "rw";
  const char kRenderedImageJsonHeightKey[] = "rh";
  const char kOriginalImageJsonWidthKey[] = "ow";
  const char kOriginalImageJsonHeightKey[] = "oh";
  const char kEmptyValuePlaceholder[] = "\n";
  
  // Create CriticalImagesInfo object from the value of property_value.  NULL if
  // no value is found, or if the property value reflects that no results are
  // available.  Result is owned by caller.
  CriticalImagesInfo* CriticalImagesInfoFromPropertyValue(
      int percent_seen_for_critical,
      const PropertyValue* property_value) {
    DCHECK(property_value != NULL);
    scoped_ptr<CriticalImagesInfo> info(new CriticalImagesInfo());
    if (!CriticalImagesFinder::PopulateCriticalImagesFromPropertyValue(
            property_value, &info->proto)) {
      return NULL;
    }
    // Fill in map fields based on proto value so that image lookups are O(lg n).
    GetCriticalKeysFromProto(percent_seen_for_critical,
                             info->proto.html_critical_image_support(),
                             &info->html_critical_images);
    GetCriticalKeysFromProto(percent_seen_for_critical,
                             info->proto.css_critical_image_support(),
                             &info->css_critical_images);
    return info.release();
  }
  
  // Setup a map for RenderedImages and their dimensions.
  void SetupRenderedImageDimensionsMap(
      const RenderedImages& rendered_images,
      RenderedImageDimensionsMap* map) {
    for (int i = 0; i < rendered_images.image_size(); ++i) {
      const RenderedImages_Image& images = rendered_images.image(i);
      // In case of beacons returning these rendered dimensions, images.src()
      // will be a hash of the image url. Hence when we do a lookup in
      // rendered_images_map we need to hash the url.
      (*map)[images.src()] = std::make_pair(
          images.rendered_width(), images.rendered_height());
    }
  }
  
  }  // namespace
  
  const char CriticalImagesFinder::kCriticalImagesPropertyName[] =
      "critical_images";
  
  const char CriticalImagesFinder::kCriticalImagesValidCount[] =
      "critical_images_valid_count";
  
  const char CriticalImagesFinder::kCriticalImagesExpiredCount[] =
      "critical_images_expired_count";
  
  const char CriticalImagesFinder::kCriticalImagesNotFoundCount[] =
      "critical_images_not_found_count";
  
  const char CriticalImagesFinder::kRenderedImageDimensionsProperty[] =
      "rendered_image_dimensions";
  
  CriticalImagesFinder::CriticalImagesFinder(const PropertyCache::Cohort* cohort,
                                             Statistics* statistics)
      : cohort_(cohort) {
    critical_images_valid_count_ = statistics->GetVariable(
        kCriticalImagesValidCount);
    critical_images_expired_count_ = statistics->GetVariable(
        kCriticalImagesExpiredCount);
    critical_images_not_found_count_ = statistics->GetVariable(
        kCriticalImagesNotFoundCount);
  }
  
  CriticalImagesFinder::~CriticalImagesFinder() {
  }
  
  void CriticalImagesFinder::InitStats(Statistics* statistics) {
    statistics->AddVariable(kCriticalImagesValidCount);
    statistics->AddVariable(kCriticalImagesExpiredCount);
    statistics->AddVariable(kCriticalImagesNotFoundCount);
  }
  
  namespace {
  
  bool IsCriticalImage(const GoogleString& image_url,
                       const StringSet& critical_images_set) {
    return (critical_images_set.find(image_url) != critical_images_set.end());
  }
  
  }  // namespace
  
  bool CriticalImagesFinder::IsHtmlCriticalImage(StringPiece image_url,
                                                 RewriteDriver* driver) {
    return IsCriticalImage(GetKeyForUrl(image_url),
                           GetHtmlCriticalImages(driver));
  }
  
  bool CriticalImagesFinder::IsCssCriticalImage(StringPiece image_url,
                                                RewriteDriver* driver) {
    return IsCriticalImage(GetKeyForUrl(image_url),
                           GetCssCriticalImages(driver));
  }
  
  bool CriticalImagesFinder::GetRenderedImageDimensions(
      RewriteDriver* driver,
      const GoogleUrl& image_src_gurl,
      std::pair<int32, int32>* dimensions) {
    UpdateCriticalImagesSetInDriver(driver);
    const CriticalImagesInfo* info = driver->critical_images_info();
    CHECK(info != NULL);
    RenderedImageDimensionsMap::const_iterator iterator =
        info->rendered_images_map.find(
            GetKeyForUrl(image_src_gurl.spec_c_str()));
    if (iterator != info->rendered_images_map.end()) {
      *dimensions = iterator->second;
      return true;
    }
    return false;
  }
  
  const StringSet& CriticalImagesFinder::GetHtmlCriticalImages(
      RewriteDriver* driver) {
    UpdateCriticalImagesSetInDriver(driver);
    const CriticalImagesInfo* info = driver->critical_images_info();
    CHECK(info != NULL);
  
    return info->html_critical_images;
  }
  
  const StringSet& CriticalImagesFinder::GetCssCriticalImages(
      RewriteDriver* driver) {
    UpdateCriticalImagesSetInDriver(driver);
    const CriticalImagesInfo* info = driver->critical_images_info();
    CHECK(info != NULL);
  
    return info->css_critical_images;
  }
  
  StringSet* CriticalImagesFinder::mutable_html_critical_images(
      RewriteDriver* driver) {
    DCHECK(driver != NULL);
    CriticalImagesInfo* driver_info = driver->critical_images_info();
    // Preserve CSS critical images if they have been updated already.
    if (driver_info == NULL) {
      driver_info = new CriticalImagesInfo;
      driver->set_critical_images_info(driver_info);
    }
    return &driver_info->html_critical_images;
  }
  
  StringSet* CriticalImagesFinder::mutable_css_critical_images(
      RewriteDriver* driver) {
    DCHECK(driver != NULL);
    CriticalImagesInfo* driver_info = driver->critical_images_info();
    // Preserve CSS critical images if they have been updated already.
    if (driver_info == NULL) {
      driver_info = new CriticalImagesInfo;
      driver->set_critical_images_info(driver_info);
    }
    return &driver_info->css_critical_images;
  }
  
  // Copy the critical images for this request from the property cache into the
  // RewriteDriver. The critical images are not stored in CriticalImageFinder
  // because the ServerContext holds the CriticalImageFinder and hence is shared
  // between requests.
  void CriticalImagesFinder::UpdateCriticalImagesSetInDriver(
      RewriteDriver* driver) {
    // Don't update critical_images_info if it's already been set.
    if (driver->critical_images_info() != NULL) {
      return;
    }
    CriticalImagesInfo* info = NULL;
    // Fallback properties can be used for critical images.
    AbstractPropertyPage* page = driver->fallback_property_page();
    if (page != NULL && cohort() != NULL) {
      PropertyValue* property_value = page->GetProperty(
          cohort(), kCriticalImagesPropertyName);
      info = ExtractCriticalImagesFromCache(driver, property_value);
      if (info != NULL) {
        info->is_critical_image_info_present = true;
        if (driver->request_context().get() != NULL) {
          driver->log_record()->SetNumHtmlCriticalImages(
              info->html_critical_images.size());
          driver->log_record()->SetNumCssCriticalImages(
              info->css_critical_images.size());
        }
      }
    }
  
    // Store an empty CriticalImagesInfo back into the driver if we don't have any
    // beacon results yet.
    if (info == NULL) {
      info = new CriticalImagesInfo;
    }
  
    if (driver->options()->Enabled(
        RewriteOptions::kResizeToRenderedImageDimensions)) {
      scoped_ptr<RenderedImages> rendered_images(
          ExtractRenderedImageDimensionsFromCache(driver));
      if (rendered_images != NULL) {
        SetupRenderedImageDimensionsMap(*rendered_images,
                                        &info->rendered_images_map);
      }
    }
  
    driver->set_critical_images_info(info);
  }
  
  bool CriticalImagesFinder::UpdateCriticalImagesCacheEntryFromDriver(
      const StringSet* html_critical_images_set,
      const StringSet* css_critical_images_set,
      RewriteDriver* driver) {
    // Update property cache if above the fold critical images are successfully
    // determined.
    // Fallback properties will be updated for critical images.
    AbstractPropertyPage* page = driver->fallback_property_page();
    return UpdateCriticalImagesCacheEntry(
        html_critical_images_set, css_critical_images_set,
        NULL /* RenderedImages Proto */,
        SupportInterval(), cohort(), page);
  }
  
  // Setup the HTML and CSS critical image sets in *critical_images using
  // *property_value.  Return true if property_value had a value, and
  // deserialization of it succeeded.
  bool CriticalImagesFinder::PopulateCriticalImagesFromPropertyValue(
      const PropertyValue* property_value,
      CriticalImages* critical_images) {
    DCHECK(property_value != NULL);
    DCHECK(critical_images != NULL);
    if (!property_value->has_value()) {
      return false;
    }
    // Check if we have the placeholder string value, indicating an empty value.
    // This will be stored when we have an empty set of critical images, since the
    // property cache doesn't store empty values.
    if (property_value->value() == kEmptyValuePlaceholder) {
      critical_images->Clear();
      return true;
    }
    // Having dealt with the unusual cases, parse the proto.
    ArrayInputStream input(property_value->value().data(),
                           property_value->value().size());
    return critical_images->ParseFromZeroCopyStream(&input);
  }
  
  bool CriticalImagesFinder::UpdateCriticalImagesCacheEntry(
      const StringSet* html_critical_images_set,
      const StringSet* css_critical_images_set,
      const RenderedImages* rendered_images_set,
      int support_interval,
      const PropertyCache::Cohort* cohort,
      AbstractPropertyPage* page) {
    // Update property cache if above the fold critical images are successfully
    // determined.
    if (page == NULL) {
      return false;
    }
    if (cohort == NULL) {
      LOG(WARNING) << "Critical Images Cohort is NULL.";
      return false;
    }
    PropertyValue* property_value = page->GetProperty(
        cohort, kCriticalImagesPropertyName);
    // Read in the current critical images, and preserve the current HTML or
    // CSS critical images if they are not being updated.
    CriticalImages critical_images;
    PopulateCriticalImagesFromPropertyValue(property_value, &critical_images);
    return UpdateAndWriteBackCriticalImagesCacheEntry(
        html_critical_images_set, css_critical_images_set, rendered_images_set,
        support_interval, cohort, page, &critical_images);
  }
  
  bool CriticalImagesFinder::UpdateAndWriteBackCriticalImagesCacheEntry(
      const StringSet* html_critical_images_set,
      const StringSet* css_critical_images_set,
      const RenderedImages* rendered_images_set,
      int support_interval,
      const PropertyCache::Cohort* cohort,
      AbstractPropertyPage* page,
      CriticalImages* critical_images) {
    // Update RenderedImages proto in property Cache.
    if (rendered_images_set != NULL) {
      UpdateInPropertyCache(
          *rendered_images_set, cohort, kRenderedImageDimensionsProperty,
          false /* don't write cohort */, page);
    }
    if (!UpdateCriticalImages(
        html_critical_images_set, css_critical_images_set,
        support_interval, critical_images)) {
      return false;
    }
  
    GoogleString buf;
    if (!critical_images->SerializeToString(&buf)) {
      LOG(WARNING) << "Serialization of critical images protobuf failed.";
      return false;
    }
    // The property cache won't store an empty value, which is what an
    // empty CriticalImages will serialize to. If buf is an empty string,
    // repalce with a placeholder that we can then handle when decoding
    // the property_cache value in
    // PopulateCriticalImagesFromPropertyValue.
    if (buf.empty()) {
      buf = kEmptyValuePlaceholder;
    }
    page->UpdateValue(cohort, kCriticalImagesPropertyName, buf);
    return true;
  }
  
  bool CriticalImagesFinder::UpdateCriticalImages(
      const StringSet* html_critical_images,
      const StringSet* css_critical_images,
      int support_interval,
      CriticalImages* critical_images) {
    DCHECK(critical_images != NULL);
    if (html_critical_images != NULL) {
      UpdateCriticalKeys(
          false /* require_prior_support */,
          *html_critical_images,
          support_interval,
          critical_images->mutable_html_critical_image_support());
    }
    if (css_critical_images != NULL) {
      UpdateCriticalKeys(
          false /* require_prior_support */,
          *css_critical_images,
          support_interval,
          critical_images->mutable_css_critical_image_support());
    }
    // We updated if either StringSet* was set.
    return (html_critical_images != NULL || css_critical_images != NULL);
  }
  
  RenderedImages* CriticalImagesFinder::ExtractRenderedImageDimensionsFromCache(
      RewriteDriver* driver) {
    PropertyCacheDecodeResult pcache_status;
    scoped_ptr<RenderedImages> dimensions(
        DecodeFromPropertyCache<RenderedImages>(
            driver,
            cohort(),
            kRenderedImageDimensionsProperty,
            driver->options()->finder_properties_cache_expiration_time_ms(),
            &pcache_status));
    if (pcache_status == kPropertyCacheDecodeParseError) {
      driver->message_handler()->Message(
          kWarning, "Unable to parse Critical RenderedImage PropertyValue for %s",
          driver->url());
    }
    return dimensions.release();
  }
  
  RenderedImages* CriticalImagesFinder::JsonMapToRenderedImagesMap(
      const GoogleString& str,
      const RewriteOptions* options) {
    try {
      Json::Reader reader;
      Json::Value json_rendered_image_map;
      if (!reader.parse(str, json_rendered_image_map)) {
        LOG(WARNING) << "Unable to parse Json data for rendered images";
        return NULL;
      }
      // Parse json data into a map.
      if (json_rendered_image_map.isNull() || !json_rendered_image_map.isObject()) {
        LOG(WARNING) << "Bad Json rendered image dimensions map";
        return NULL;
      }
      // Put the extracted map into RenderedImages proto data.
      RenderedImages* rendered_images = new RenderedImages();
      Json::Value::Members imgs = json_rendered_image_map.getMemberNames();
      for (int i = 0, n = imgs.size(); i < n; ++i) {
        const GoogleString& img_src = imgs[i];
        int original_width = json_rendered_image_map[img_src].get(
            kOriginalImageJsonWidthKey, 0).asInt();
        int original_height = json_rendered_image_map[img_src].get(
            kOriginalImageJsonHeightKey, 0).asInt();
        int rendered_width = json_rendered_image_map[img_src].get(
            kRenderedImageJsonWidthKey, 0).asInt();
        int rendered_height = json_rendered_image_map[img_src].get(
            kRenderedImageJsonHeightKey, 0).asInt();
        int original_area = (original_width * original_height);
        int rendered_area = (rendered_width * rendered_height);
        // Store renderedWidth and renderedHeight for the image only if
        // the rendered sizes are lower than the original sizes by at least the
        // percentage threshold set.
        if (100 * rendered_area < original_area *
            options->image_limit_rendered_area_percent()) {
          RenderedImages_Image* images = rendered_images->add_image();
          images->set_src(img_src);
          images->set_rendered_width(rendered_width);
          images->set_rendered_height(rendered_height);
        }
      }
      return rendered_images;
    } catch (std::exception& e) {
      LOG(WARNING) << "Bad Json rendered image dimensions map";
      return NULL;
    }
  }
  
  CriticalImagesInfo* CriticalImagesFinder::ExtractCriticalImagesFromCache(
      RewriteDriver* driver,
      const PropertyValue* property_value) {
    CriticalImagesInfo* critical_images_info = NULL;
    const PropertyCache* page_property_cache =
        driver->server_context()->page_property_cache();
    int64 cache_ttl_ms =
        driver->options()->finder_properties_cache_expiration_time_ms();
    // Check if the cache value exists and is not expired.
    if (property_value->has_value()) {
      const bool is_valid =
          !page_property_cache->IsExpired(property_value, cache_ttl_ms);
      if (is_valid) {
        critical_images_info =
            CriticalImagesInfoFromPropertyValue(PercentSeenForCritical(),
                                                property_value);
        if (critical_images_info == NULL) {
          critical_images_not_found_count_->Add(1);
        } else {
          critical_images_valid_count_->Add(1);
        }
      } else {
        critical_images_expired_count_->Add(1);
      }
    } else {
      critical_images_not_found_count_->Add(1);
    }
    return critical_images_info;
  }
  
  CriticalImagesFinder::Availability CriticalImagesFinder::Available(
      RewriteDriver* driver) {
    UpdateCriticalImagesSetInDriver(driver);
    CriticalImagesInfo* info = driver->critical_images_info();
    if (info != NULL && info->is_critical_image_info_present &&
        info->proto.has_html_critical_image_support() &&
        IsBeaconDataAvailable(info->proto.html_critical_image_support())) {
      return kAvailable;
    } else {
      return kNoDataYet;
    }
  }
  
  bool CriticalImagesFinder::IsCriticalImageInfoPresent(RewriteDriver* driver) {
    UpdateCriticalImagesSetInDriver(driver);
    return driver->critical_images_info()->is_critical_image_info_present;
  }
  
  void CriticalImagesFinder::AddHtmlCriticalImage(
      const GoogleString& url,
      RewriteDriver* driver) {
    mutable_html_critical_images(driver)->insert(GetKeyForUrl(url));
  }
  
  }  // namespace net_instaweb
  