palemoon-dev/platform/netwerk/base/Predictor.cpp

/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

#include <algorithm>

#include "Predictor.h"

#include "nsAppDirectoryServiceDefs.h"
#include "nsICacheStorage.h"
#include "nsICacheStorageService.h"
#include "nsICachingChannel.h"
#include "nsICancelable.h"
#include "nsIChannel.h"
#include "nsContentUtils.h"
#include "nsIDNSService.h"
#include "nsIDocument.h"
#include "nsIFile.h"
#include "nsIHttpChannel.h"
#include "nsIInputStream.h"
#include "nsIIOService.h"
#include "nsILoadContext.h"
#include "nsILoadContextInfo.h"
#include "nsILoadGroup.h"
#include "nsINetworkPredictorVerifier.h"
#include "nsIObserverService.h"
#include "nsIPrefBranch.h"
#include "nsIPrefService.h"
#include "nsISpeculativeConnect.h"
#include "nsITimer.h"
#include "nsIURI.h"
#include "nsNetUtil.h"
#include "nsServiceManagerUtils.h"
#include "nsStreamUtils.h"
#include "nsString.h"
#include "nsThreadUtils.h"
#include "nsHttpRequestHead.h"
#include "mozilla/Logging.h"

#include "mozilla/Preferences.h"
#include "mozilla/Telemetry.h"

#include "mozilla/net/NeckoCommon.h"
#include "mozilla/net/NeckoParent.h"

#include "LoadContextInfo.h"
#include "mozilla/ipc/URIUtils.h"
#include "SerializedLoadContext.h"
#include "mozilla/net/NeckoChild.h"

#include "mozilla/dom/ContentParent.h"

using namespace mozilla;
using namespace mozilla::dom;

namespace mozilla {
namespace net {

Predictor *Predictor::sSelf = nullptr;

static LazyLogModule gPredictorLog("NetworkPredictor");

#define PREDICTOR_LOG(args) MOZ_LOG(gPredictorLog, mozilla::LogLevel::Debug, args)

#define RETURN_IF_FAILED(_rv) \
  do { \
    if (NS_FAILED(_rv)) { \
      return; \
    } \
  } while (0)

#define NOW_IN_SECONDS() static_cast<uint32_t>(PR_Now() / PR_USEC_PER_SEC)


static const char PREDICTOR_ENABLED_PREF[] = "network.predictor.enabled";
static const char PREDICTOR_SSL_HOVER_PREF[] = "network.predictor.enable-hover-on-ssl";
static const char PREDICTOR_PREFETCH_PREF[] = "network.predictor.enable-prefetch";

static const char PREDICTOR_PAGE_DELTA_DAY_PREF[] =
  "network.predictor.page-degradation.day";
static const int32_t PREDICTOR_PAGE_DELTA_DAY_DEFAULT = 0;
static const char PREDICTOR_PAGE_DELTA_WEEK_PREF[] =
  "network.predictor.page-degradation.week";
static const int32_t PREDICTOR_PAGE_DELTA_WEEK_DEFAULT = 5;
static const char PREDICTOR_PAGE_DELTA_MONTH_PREF[] =
  "network.predictor.page-degradation.month";
static const int32_t PREDICTOR_PAGE_DELTA_MONTH_DEFAULT = 10;
static const char PREDICTOR_PAGE_DELTA_YEAR_PREF[] =
  "network.predictor.page-degradation.year";
static const int32_t PREDICTOR_PAGE_DELTA_YEAR_DEFAULT = 25;
static const char PREDICTOR_PAGE_DELTA_MAX_PREF[] =
  "network.predictor.page-degradation.max";
static const int32_t PREDICTOR_PAGE_DELTA_MAX_DEFAULT = 50;
static const char PREDICTOR_SUB_DELTA_DAY_PREF[] =
  "network.predictor.subresource-degradation.day";
static const int32_t PREDICTOR_SUB_DELTA_DAY_DEFAULT = 1;
static const char PREDICTOR_SUB_DELTA_WEEK_PREF[] =
  "network.predictor.subresource-degradation.week";
static const int32_t PREDICTOR_SUB_DELTA_WEEK_DEFAULT = 10;
static const char PREDICTOR_SUB_DELTA_MONTH_PREF[] =
  "network.predictor.subresource-degradation.month";
static const int32_t PREDICTOR_SUB_DELTA_MONTH_DEFAULT = 25;
static const char PREDICTOR_SUB_DELTA_YEAR_PREF[] =
  "network.predictor.subresource-degradation.year";
static const int32_t PREDICTOR_SUB_DELTA_YEAR_DEFAULT = 50;
static const char PREDICTOR_SUB_DELTA_MAX_PREF[] =
  "network.predictor.subresource-degradation.max";
static const int32_t PREDICTOR_SUB_DELTA_MAX_DEFAULT = 100;

static const char PREDICTOR_PREFETCH_ROLLING_LOAD_PREF[] =
  "network.predictor.prefetch-rolling-load-count";
static const int32_t PREFETCH_ROLLING_LOAD_DEFAULT = 10;
static const char PREDICTOR_PREFETCH_MIN_PREF[] =
  "network.predictor.prefetch-min-confidence";
static const int32_t PREFETCH_MIN_DEFAULT = 100;
static const char PREDICTOR_PRECONNECT_MIN_PREF[] =
  "network.predictor.preconnect-min-confidence";
static const int32_t PRECONNECT_MIN_DEFAULT = 90;
static const char PREDICTOR_PRERESOLVE_MIN_PREF[] =
  "network.predictor.preresolve-min-confidence";
static const int32_t PRERESOLVE_MIN_DEFAULT = 60;
static const char PREDICTOR_REDIRECT_LIKELY_PREF[] =
  "network.predictor.redirect-likely-confidence";
static const int32_t REDIRECT_LIKELY_DEFAULT = 75;

static const char PREDICTOR_PREFETCH_FORCE_VALID_PREF[] =
  "network.predictor.prefetch-force-valid-for";
static const int32_t PREFETCH_FORCE_VALID_DEFAULT = 10;

static const char PREDICTOR_MAX_RESOURCES_PREF[] =
  "network.predictor.max-resources-per-entry";
static const uint32_t PREDICTOR_MAX_RESOURCES_DEFAULT = 100;

// This is selected in concert with max-resources-per-entry to keep memory usage
// low-ish. The default of the combo of the two is ~50k
static const char PREDICTOR_MAX_URI_LENGTH_PREF[] =
  "network.predictor.max-uri-length";
static const uint32_t PREDICTOR_MAX_URI_LENGTH_DEFAULT = 500;

static const char PREDICTOR_DOING_TESTS_PREF[] = "network.predictor.doing-tests";

static const char PREDICTOR_CLEANED_UP_PREF[] = "network.predictor.cleaned-up";

// All these time values are in sec
static const uint32_t ONE_DAY = 86400U;
static const uint32_t ONE_WEEK = 7U * ONE_DAY;
static const uint32_t ONE_MONTH = 30U * ONE_DAY;
static const uint32_t ONE_YEAR = 365U * ONE_DAY;

static const uint32_t STARTUP_WINDOW = 5U * 60U; // 5min

// Version of metadata entries we expect
static const uint32_t METADATA_VERSION = 1;

// Flags available in entries
// FLAG_PREFETCHABLE - we have determined that this item is eligible for prefetch
static const uint32_t FLAG_PREFETCHABLE = 1 << 0;

// We save 12 bits in the "flags" section of our metadata for actual flags, the
// rest are to keep track of a rolling count of which loads a resource has been
// used on to determine if we can prefetch that resource or not;
static const uint8_t kRollingLoadOffset = 12;
static const int32_t kMaxPrefetchRollingLoadCount = 20;
static const uint32_t kFlagsMask = ((1 << kRollingLoadOffset) - 1);

// ID Extensions for cache entries
#define PREDICTOR_ORIGIN_EXTENSION "predictor-origin"

// Get the full origin (scheme, host, port) out of a URI (maybe should be part
// of nsIURI instead?)
static nsresult
ExtractOrigin(nsIURI *uri, nsIURI **originUri, nsIIOService *ioService)
{
  nsAutoCString s;
  s.Truncate();
  nsresult rv = nsContentUtils::GetASCIIOrigin(uri, s);
  NS_ENSURE_SUCCESS(rv, rv);

  return NS_NewURI(originUri, s, nullptr, nullptr, ioService);
}

// All URIs we get passed *must* be http or https if they're not null. This
// helps ensure that.
static bool
IsNullOrHttp(nsIURI *uri)
{
  if (!uri) {
    return true;
  }

  bool isHTTP = false;
  uri->SchemeIs("http", &isHTTP);
  if (!isHTTP) {
    uri->SchemeIs("https", &isHTTP);
  }

  return isHTTP;
}

// Listener for the speculative DNS requests we'll fire off, which just ignores
// the result (since we're just trying to warm the cache). This also exists to
// reduce round-trips to the main thread, by being something threadsafe the
// Predictor can use.

NS_IMPL_ISUPPORTS(Predictor::DNSListener, nsIDNSListener);

NS_IMETHODIMP
Predictor::DNSListener::OnLookupComplete(nsICancelable *request,
                                         nsIDNSRecord *rec,
                                         nsresult status)
{
  return NS_OK;
}

// Class to proxy important information from the initial predictor call through
// the cache API and back into the internals of the predictor. We can't use the
// predictor itself, as it may have multiple actions in-flight, and each action
// has different parameters.
NS_IMPL_ISUPPORTS(Predictor::Action, nsICacheEntryOpenCallback);

Predictor::Action::Action(bool fullUri, bool predict,
                          Predictor::Reason reason,
                          nsIURI *targetURI, nsIURI *sourceURI,
                          nsINetworkPredictorVerifier *verifier,
                          Predictor *predictor)
  :mFullUri(fullUri)
  ,mPredict(predict)
  ,mTargetURI(targetURI)
  ,mSourceURI(sourceURI)
  ,mVerifier(verifier)
  ,mStackCount(0)
  ,mPredictor(predictor)
{
  mStartTime = TimeStamp::Now();
  if (mPredict) {
    mPredictReason = reason.mPredict;
  } else {
    mLearnReason = reason.mLearn;
  }
}

Predictor::Action::Action(bool fullUri, bool predict,
                          Predictor::Reason reason,
                          nsIURI *targetURI, nsIURI *sourceURI,
                          nsINetworkPredictorVerifier *verifier,
                          Predictor *predictor, uint8_t stackCount)
  :mFullUri(fullUri)
  ,mPredict(predict)
  ,mTargetURI(targetURI)
  ,mSourceURI(sourceURI)
  ,mVerifier(verifier)
  ,mStackCount(stackCount)
  ,mPredictor(predictor)
{
  mStartTime = TimeStamp::Now();
  if (mPredict) {
    mPredictReason = reason.mPredict;
  } else {
    mLearnReason = reason.mLearn;
  }
}

Predictor::Action::~Action()
{ }

NS_IMETHODIMP
Predictor::Action::OnCacheEntryCheck(nsICacheEntry *entry,
                                     nsIApplicationCache *appCache,
                                     uint32_t *result)
{
  *result = nsICacheEntryOpenCallback::ENTRY_WANTED;
  return NS_OK;
}

NS_IMETHODIMP
Predictor::Action::OnCacheEntryAvailable(nsICacheEntry *entry, bool isNew,
                                         nsIApplicationCache *appCache,
                                         nsresult result)
{
  MOZ_ASSERT(NS_IsMainThread(), "Got cache entry off main thread!");

  nsAutoCString targetURI, sourceURI;
  mTargetURI->GetAsciiSpec(targetURI);
  if (mSourceURI) {
    mSourceURI->GetAsciiSpec(sourceURI);
  }
  PREDICTOR_LOG(("OnCacheEntryAvailable %p called. entry=%p mFullUri=%d mPredict=%d "
                 "mPredictReason=%d mLearnReason=%d mTargetURI=%s "
                 "mSourceURI=%s mStackCount=%d isNew=%d result=0x%08x",
                 this, entry, mFullUri, mPredict, mPredictReason, mLearnReason,
                 targetURI.get(), sourceURI.get(), mStackCount,
                 isNew, result));
  if (NS_FAILED(result)) {
    PREDICTOR_LOG(("OnCacheEntryAvailable %p FAILED to get cache entry (0x%08X). "
                   "Aborting.", this, result));
    return NS_OK;
  }
  if (!mPredict) {
    mPredictor->LearnInternal(mLearnReason, entry, isNew, mFullUri, mTargetURI,
                              mSourceURI);
  }

  return NS_OK;
}

NS_IMPL_ISUPPORTS(Predictor,
                  nsINetworkPredictor,
                  nsIObserver,
                  nsISpeculativeConnectionOverrider,
                  nsIInterfaceRequestor,
                  nsICacheEntryMetaDataVisitor,
                  nsINetworkPredictorVerifier)

Predictor::Predictor()
  :mInitialized(false)
  ,mEnabled(true)
  ,mEnableHoverOnSSL(false)
  ,mEnablePrefetch(true)
  ,mPageDegradationDay(PREDICTOR_PAGE_DELTA_DAY_DEFAULT)
  ,mPageDegradationWeek(PREDICTOR_PAGE_DELTA_WEEK_DEFAULT)
  ,mPageDegradationMonth(PREDICTOR_PAGE_DELTA_MONTH_DEFAULT)
  ,mPageDegradationYear(PREDICTOR_PAGE_DELTA_YEAR_DEFAULT)
  ,mPageDegradationMax(PREDICTOR_PAGE_DELTA_MAX_DEFAULT)
  ,mSubresourceDegradationDay(PREDICTOR_SUB_DELTA_DAY_DEFAULT)
  ,mSubresourceDegradationWeek(PREDICTOR_SUB_DELTA_WEEK_DEFAULT)
  ,mSubresourceDegradationMonth(PREDICTOR_SUB_DELTA_MONTH_DEFAULT)
  ,mSubresourceDegradationYear(PREDICTOR_SUB_DELTA_YEAR_DEFAULT)
  ,mSubresourceDegradationMax(PREDICTOR_SUB_DELTA_MAX_DEFAULT)
  ,mPrefetchRollingLoadCount(PREFETCH_ROLLING_LOAD_DEFAULT)
  ,mPrefetchMinConfidence(PREFETCH_MIN_DEFAULT)
  ,mPreconnectMinConfidence(PRECONNECT_MIN_DEFAULT)
  ,mPreresolveMinConfidence(PRERESOLVE_MIN_DEFAULT)
  ,mRedirectLikelyConfidence(REDIRECT_LIKELY_DEFAULT)
  ,mPrefetchForceValidFor(PREFETCH_FORCE_VALID_DEFAULT)
  ,mMaxResourcesPerEntry(PREDICTOR_MAX_RESOURCES_DEFAULT)
  ,mStartupCount(1)
  ,mMaxURILength(PREDICTOR_MAX_URI_LENGTH_DEFAULT)
  ,mDoingTests(false)
{
  MOZ_ASSERT(!sSelf, "multiple Predictor instances!");
  sSelf = this;
}

Predictor::~Predictor()
{
  if (mInitialized)
    Shutdown();

  sSelf = nullptr;
}

// Predictor::nsIObserver

nsresult
Predictor::InstallObserver()
{
  MOZ_ASSERT(NS_IsMainThread(), "Installing observer off main thread");

  nsresult rv = NS_OK;
  nsCOMPtr<nsIObserverService> obs =
    mozilla::services::GetObserverService();
  if (!obs) {
    return NS_ERROR_NOT_AVAILABLE;
  }

  rv = obs->AddObserver(this, NS_XPCOM_SHUTDOWN_OBSERVER_ID, false);
  NS_ENSURE_SUCCESS(rv, rv);

  Preferences::AddBoolVarCache(&mEnabled, PREDICTOR_ENABLED_PREF, true);
  Preferences::AddBoolVarCache(&mEnableHoverOnSSL,
                               PREDICTOR_SSL_HOVER_PREF, false);
  Preferences::AddBoolVarCache(&mEnablePrefetch, PREDICTOR_PREFETCH_PREF, true);
  Preferences::AddIntVarCache(&mPageDegradationDay,
                              PREDICTOR_PAGE_DELTA_DAY_PREF,
                              PREDICTOR_PAGE_DELTA_DAY_DEFAULT);
  Preferences::AddIntVarCache(&mPageDegradationWeek,
                              PREDICTOR_PAGE_DELTA_WEEK_PREF,
                              PREDICTOR_PAGE_DELTA_WEEK_DEFAULT);
  Preferences::AddIntVarCache(&mPageDegradationMonth,
                              PREDICTOR_PAGE_DELTA_MONTH_PREF,
                              PREDICTOR_PAGE_DELTA_MONTH_DEFAULT);
  Preferences::AddIntVarCache(&mPageDegradationYear,
                              PREDICTOR_PAGE_DELTA_YEAR_PREF,
                              PREDICTOR_PAGE_DELTA_YEAR_DEFAULT);
  Preferences::AddIntVarCache(&mPageDegradationMax,
                              PREDICTOR_PAGE_DELTA_MAX_PREF,
                              PREDICTOR_PAGE_DELTA_MAX_DEFAULT);

  Preferences::AddIntVarCache(&mSubresourceDegradationDay,
                              PREDICTOR_SUB_DELTA_DAY_PREF,
                              PREDICTOR_SUB_DELTA_DAY_DEFAULT);
  Preferences::AddIntVarCache(&mSubresourceDegradationWeek,
                              PREDICTOR_SUB_DELTA_WEEK_PREF,
                              PREDICTOR_SUB_DELTA_WEEK_DEFAULT);
  Preferences::AddIntVarCache(&mSubresourceDegradationMonth,
                              PREDICTOR_SUB_DELTA_MONTH_PREF,
                              PREDICTOR_SUB_DELTA_MONTH_DEFAULT);
  Preferences::AddIntVarCache(&mSubresourceDegradationYear,
                              PREDICTOR_SUB_DELTA_YEAR_PREF,
                              PREDICTOR_SUB_DELTA_YEAR_DEFAULT);
  Preferences::AddIntVarCache(&mSubresourceDegradationMax,
                              PREDICTOR_SUB_DELTA_MAX_PREF,
                              PREDICTOR_SUB_DELTA_MAX_DEFAULT);

  Preferences::AddIntVarCache(&mPrefetchRollingLoadCount,
                              PREDICTOR_PREFETCH_ROLLING_LOAD_PREF,
                              PREFETCH_ROLLING_LOAD_DEFAULT);
  Preferences::AddIntVarCache(&mPrefetchMinConfidence,
                              PREDICTOR_PREFETCH_MIN_PREF,
                              PREFETCH_MIN_DEFAULT);
  Preferences::AddIntVarCache(&mPreconnectMinConfidence,
                              PREDICTOR_PRECONNECT_MIN_PREF,
                              PRECONNECT_MIN_DEFAULT);
  Preferences::AddIntVarCache(&mPreresolveMinConfidence,
                              PREDICTOR_PRERESOLVE_MIN_PREF,
                              PRERESOLVE_MIN_DEFAULT);
  Preferences::AddIntVarCache(&mRedirectLikelyConfidence,
                              PREDICTOR_REDIRECT_LIKELY_PREF,
                              REDIRECT_LIKELY_DEFAULT);

  Preferences::AddIntVarCache(&mPrefetchForceValidFor,
                              PREDICTOR_PREFETCH_FORCE_VALID_PREF,
                              PREFETCH_FORCE_VALID_DEFAULT);

  Preferences::AddIntVarCache(&mMaxResourcesPerEntry,
                              PREDICTOR_MAX_RESOURCES_PREF,
                              PREDICTOR_MAX_RESOURCES_DEFAULT);

  Preferences::AddBoolVarCache(&mCleanedUp, PREDICTOR_CLEANED_UP_PREF, false);

  Preferences::AddUintVarCache(&mMaxURILength, PREDICTOR_MAX_URI_LENGTH_PREF,
                               PREDICTOR_MAX_URI_LENGTH_DEFAULT);

  Preferences::AddBoolVarCache(&mDoingTests, PREDICTOR_DOING_TESTS_PREF, false);

  if (!mCleanedUp) {
    mCleanupTimer = do_CreateInstance("@mozilla.org/timer;1");
    mCleanupTimer->Init(this, 60 * 1000, nsITimer::TYPE_ONE_SHOT);
  }

  return rv;
}

void
Predictor::RemoveObserver()
{
  MOZ_ASSERT(NS_IsMainThread(), "Removing observer off main thread");

  nsCOMPtr<nsIObserverService> obs =
    mozilla::services::GetObserverService();
  if (obs) {
    obs->RemoveObserver(this, NS_XPCOM_SHUTDOWN_OBSERVER_ID);
  }

  if (mCleanupTimer) {
    mCleanupTimer->Cancel();
    mCleanupTimer = nullptr;
  }
}

NS_IMETHODIMP
Predictor::Observe(nsISupports *subject, const char *topic,
                   const char16_t *data_unicode)
{
  nsresult rv = NS_OK;
  MOZ_ASSERT(NS_IsMainThread(),
             "Predictor observing something off main thread!");

  if (!strcmp(NS_XPCOM_SHUTDOWN_OBSERVER_ID, topic)) {
    Shutdown();
  } else if (!strcmp("timer-callback", topic)) {
    MaybeCleanupOldDBFiles();
    mCleanupTimer = nullptr;
  }

  return rv;
}

// Predictor::nsISpeculativeConnectionOverrider

NS_IMETHODIMP
Predictor::GetIgnoreIdle(bool *ignoreIdle)
{
  *ignoreIdle = true;
  return NS_OK;
}

NS_IMETHODIMP
Predictor::GetParallelSpeculativeConnectLimit(
    uint32_t *parallelSpeculativeConnectLimit)
{
  *parallelSpeculativeConnectLimit = 6;
  return NS_OK;
}

NS_IMETHODIMP
Predictor::GetIsFromPredictor(bool *isFromPredictor)
{
  *isFromPredictor = true;
  return NS_OK;
}

NS_IMETHODIMP
Predictor::GetAllow1918(bool *allow1918)
{
  *allow1918 = false;
  return NS_OK;
}

// Predictor::nsIInterfaceRequestor

NS_IMETHODIMP
Predictor::GetInterface(const nsIID &iid, void **result)
{
  return QueryInterface(iid, result);
}

// Predictor::nsICacheEntryMetaDataVisitor

#define SEEN_META_DATA "predictor::seen"
#define RESOURCE_META_DATA "predictor::resource-count"
#define META_DATA_PREFIX "predictor::"

static bool
IsURIMetadataElement(const char *key)
{
  return StringBeginsWith(nsDependentCString(key),
                          NS_LITERAL_CSTRING(META_DATA_PREFIX)) &&
         !NS_LITERAL_CSTRING(SEEN_META_DATA).Equals(key) &&
         !NS_LITERAL_CSTRING(RESOURCE_META_DATA).Equals(key);
}

nsresult
Predictor::OnMetaDataElement(const char *asciiKey, const char *asciiValue)
{
  MOZ_ASSERT(NS_IsMainThread());

  if (!IsURIMetadataElement(asciiKey)) {
    // This isn't a bit of metadata we care about
    return NS_OK;
  }

  nsCString key, value;
  key.AssignASCII(asciiKey);
  value.AssignASCII(asciiValue);
  mKeysToOperateOn.AppendElement(key);
  mValuesToOperateOn.AppendElement(value);

  return NS_OK;
}

// Predictor::nsINetworkPredictor

nsresult
Predictor::Init()
{
  MOZ_DIAGNOSTIC_ASSERT(!IsNeckoChild());

  if (!NS_IsMainThread()) {
    MOZ_ASSERT(false, "Predictor::Init called off the main thread!");
    return NS_ERROR_UNEXPECTED;
  }

  nsresult rv = NS_OK;

  rv = InstallObserver();
  NS_ENSURE_SUCCESS(rv, rv);

  mLastStartupTime = mStartupTime = NOW_IN_SECONDS();

  if (!mDNSListener) {
    mDNSListener = new DNSListener();
  }

  nsCOMPtr<nsICacheStorageService> cacheStorageService =
    do_GetService("@mozilla.org/netwerk/cache-storage-service;1", &rv);
  NS_ENSURE_SUCCESS(rv, rv);

  RefPtr<LoadContextInfo> lci =
    new LoadContextInfo(false, NeckoOriginAttributes());

  rv = cacheStorageService->DiskCacheStorage(lci, false,
                                             getter_AddRefs(mCacheDiskStorage));
  NS_ENSURE_SUCCESS(rv, rv);

  mIOService = do_GetService("@mozilla.org/network/io-service;1", &rv);
  NS_ENSURE_SUCCESS(rv, rv);

  rv = NS_NewURI(getter_AddRefs(mStartupURI),
                 "predictor://startup", nullptr, mIOService);
  NS_ENSURE_SUCCESS(rv, rv);

  mSpeculativeService = do_QueryInterface(mIOService, &rv);
  NS_ENSURE_SUCCESS(rv, rv);

  mDnsService = do_GetService("@mozilla.org/network/dns-service;1", &rv);
  NS_ENSURE_SUCCESS(rv, rv);

  mInitialized = true;

  return rv;
}

namespace {
class PredictorThreadShutdownRunner : public Runnable
{
public:
  PredictorThreadShutdownRunner(nsIThread *ioThread, bool success)
    :mIOThread(ioThread)
    ,mSuccess(success)
  { }
  ~PredictorThreadShutdownRunner() { }

  NS_IMETHOD Run() override
  {
    MOZ_ASSERT(NS_IsMainThread(), "Shutting down io thread off main thread!");
    if (mSuccess) {
      // This means the cleanup happened. Mark so we don't try in the
      // future.
      Preferences::SetBool(PREDICTOR_CLEANED_UP_PREF, true);
    }
    return mIOThread->AsyncShutdown();
  }

private:
  nsCOMPtr<nsIThread> mIOThread;
  bool mSuccess;
};

class PredictorOldCleanupRunner : public Runnable
{
public:
  PredictorOldCleanupRunner(nsIThread *ioThread, nsIFile *dbFile)
    :mIOThread(ioThread)
    ,mDBFile(dbFile)
  { }

  ~PredictorOldCleanupRunner() { }

  NS_IMETHOD Run() override
  {
    MOZ_ASSERT(!NS_IsMainThread(), "Cleaning up old files on main thread!");
    nsresult rv = CheckForAndDeleteOldDBFiles();
    RefPtr<PredictorThreadShutdownRunner> runner =
      new PredictorThreadShutdownRunner(mIOThread, NS_SUCCEEDED(rv));
    NS_DispatchToMainThread(runner);
    return NS_OK;
  }

private:
  nsresult CheckForAndDeleteOldDBFiles()
  {
    nsCOMPtr<nsIFile> oldDBFile;
    nsresult rv = mDBFile->GetParent(getter_AddRefs(oldDBFile));
    NS_ENSURE_SUCCESS(rv, rv);

    rv = oldDBFile->AppendNative(NS_LITERAL_CSTRING("seer.sqlite"));
    NS_ENSURE_SUCCESS(rv, rv);

    bool fileExists = false;
    rv = oldDBFile->Exists(&fileExists);
    NS_ENSURE_SUCCESS(rv, rv);

    if (fileExists) {
      rv = oldDBFile->Remove(false);
      NS_ENSURE_SUCCESS(rv, rv);
    }

    fileExists = false;
    rv = mDBFile->Exists(&fileExists);
    NS_ENSURE_SUCCESS(rv, rv);

    if (fileExists) {
      rv = mDBFile->Remove(false);
    }

    return rv;
  }

  nsCOMPtr<nsIThread> mIOThread;
  nsCOMPtr<nsIFile> mDBFile;
};

} // namespace

void
Predictor::MaybeCleanupOldDBFiles()
{
  MOZ_ASSERT(NS_IsMainThread());

  if (!mEnabled || mCleanedUp) {
    return;
  }

  mCleanedUp = true;

  // This is used for cleaning up junk left over from the old backend
  // built on top of sqlite, if necessary.
  nsCOMPtr<nsIFile> dbFile;
  nsresult rv = NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR,
                                       getter_AddRefs(dbFile));
  RETURN_IF_FAILED(rv);
  rv = dbFile->AppendNative(NS_LITERAL_CSTRING("netpredictions.sqlite"));
  RETURN_IF_FAILED(rv);

  nsCOMPtr<nsIThread> ioThread;
  rv = NS_NewNamedThread("NetPredictClean", getter_AddRefs(ioThread));
  RETURN_IF_FAILED(rv);

  RefPtr<PredictorOldCleanupRunner> runner =
    new PredictorOldCleanupRunner(ioThread, dbFile);
  ioThread->Dispatch(runner, NS_DISPATCH_NORMAL);
}

void
Predictor::Shutdown()
{
  if (!NS_IsMainThread()) {
    MOZ_ASSERT(false, "Predictor::Shutdown called off the main thread!");
    return;
  }

  RemoveObserver();

  mInitialized = false;
}

nsresult
Predictor::Create(nsISupports *aOuter, const nsIID& aIID,
                  void **aResult)
{
  MOZ_ASSERT(NS_IsMainThread());

  nsresult rv;

  if (aOuter != nullptr) {
    return NS_ERROR_NO_AGGREGATION;
  }

  RefPtr<Predictor> svc = new Predictor();
  if (IsNeckoChild()) {
    // Child threads only need to be call into the public interface methods
    // so we don't bother with initialization
    return svc->QueryInterface(aIID, aResult);
  }

  rv = svc->Init();
  if (NS_FAILED(rv)) {
    PREDICTOR_LOG(("Failed to initialize predictor, predictor will be a noop"));
  }

  // We treat init failure the same as the service being disabled, since this
  // is all an optimization anyway. No need to freak people out. That's why we
  // gladly continue on QI'ing here.
  rv = svc->QueryInterface(aIID, aResult);

  return rv;
}

// Called from the main thread to initiate predictive actions
NS_IMETHODIMP
Predictor::Predict(nsIURI *targetURI, nsIURI *sourceURI,
                   PredictorPredictReason reason, nsILoadContext *loadContext,
                   nsINetworkPredictorVerifier *verifier)
{
  MOZ_ASSERT(NS_IsMainThread(),
             "Predictor interface methods must be called on the main thread");

  PREDICTOR_LOG(("Predictor::Predict"));

  if (IsNeckoChild()) {
    MOZ_DIAGNOSTIC_ASSERT(gNeckoChild);

    PREDICTOR_LOG(("    called on child process"));

    ipc::OptionalURIParams serTargetURI, serSourceURI;
    SerializeURI(targetURI, serTargetURI);
    SerializeURI(sourceURI, serSourceURI);

    IPC::SerializedLoadContext serLoadContext;
    serLoadContext.Init(loadContext);

    // If two different threads are predicting concurently, this will be
    // overwritten. Thankfully, we only use this in tests, which will
    // overwrite mVerifier perhaps multiple times for each individual test;
    // however, within each test, the multiple predict calls should have the
    // same verifier.
    if (verifier) {
      PREDICTOR_LOG(("    was given a verifier"));
      mChildVerifier = verifier;
    }
    PREDICTOR_LOG(("    forwarding to parent process"));
    gNeckoChild->SendPredPredict(serTargetURI, serSourceURI,
                                 reason, serLoadContext, verifier);
    return NS_OK;
  }

  PREDICTOR_LOG(("    called on parent process"));

  if (!mInitialized) {
    PREDICTOR_LOG(("    not initialized"));
    return NS_OK;
  }

  if (!mEnabled) {
    PREDICTOR_LOG(("    not enabled"));
    return NS_OK;
  }

  if (loadContext && loadContext->UsePrivateBrowsing()) {
    // Don't want to do anything in PB mode
    PREDICTOR_LOG(("    in PB mode"));
    return NS_OK;
  }

  if (!IsNullOrHttp(targetURI) || !IsNullOrHttp(sourceURI)) {
    // Nothing we can do for non-HTTP[S] schemes
    PREDICTOR_LOG(("    got non-http[s] URI"));
    return NS_OK;
  }

  // Ensure we've been given the appropriate arguments for the kind of
  // prediction we're being asked to do
  nsCOMPtr<nsIURI> uriKey = targetURI;
  nsCOMPtr<nsIURI> originKey;
  switch (reason) {
    case nsINetworkPredictor::PREDICT_LINK:
      if (!targetURI || !sourceURI) {
        PREDICTOR_LOG(("    link invalid URI state"));
        return NS_ERROR_INVALID_ARG;
      }
      // Link hover is a special case where we can predict without hitting the
      // db, so let's go ahead and fire off that prediction here.
      PredictForLink(targetURI, sourceURI, verifier);
      return NS_OK;
    case nsINetworkPredictor::PREDICT_LOAD:
      if (!targetURI || sourceURI) {
        PREDICTOR_LOG(("    load invalid URI state"));
        return NS_ERROR_INVALID_ARG;
      }
      break;
    case nsINetworkPredictor::PREDICT_STARTUP:
      if (targetURI || sourceURI) {
        PREDICTOR_LOG(("    startup invalid URI state"));
        return NS_ERROR_INVALID_ARG;
      }
      uriKey = mStartupURI;
      originKey = mStartupURI;
      break;
    default:
      PREDICTOR_LOG(("    invalid reason"));
      return NS_ERROR_INVALID_ARG;
  }

  Predictor::Reason argReason;
  argReason.mPredict = reason;

  // First we open the regular cache entry, to ensure we don't gum up the works
  // waiting on the less-important predictor-only cache entry
  RefPtr<Predictor::Action> uriAction =
    new Predictor::Action(Predictor::Action::IS_FULL_URI,
                          Predictor::Action::DO_PREDICT, argReason, targetURI,
                          nullptr, verifier, this);
  nsAutoCString uriKeyStr;
  uriKey->GetAsciiSpec(uriKeyStr);
  PREDICTOR_LOG(("    Predict uri=%s reason=%d action=%p", uriKeyStr.get(),
                 reason, uriAction.get()));
  uint32_t openFlags = nsICacheStorage::OPEN_READONLY |
                       nsICacheStorage::OPEN_SECRETLY |
                       nsICacheStorage::OPEN_PRIORITY |
                       nsICacheStorage::CHECK_MULTITHREADED;
  mCacheDiskStorage->AsyncOpenURI(uriKey, EmptyCString(), openFlags, uriAction);

  // Now we do the origin-only (and therefore predictor-only) entry
  nsCOMPtr<nsIURI> targetOrigin;
  nsresult rv = ExtractOrigin(uriKey, getter_AddRefs(targetOrigin), mIOService);
  NS_ENSURE_SUCCESS(rv, rv);
  if (!originKey) {
    originKey = targetOrigin;
  }

  RefPtr<Predictor::Action> originAction =
    new Predictor::Action(Predictor::Action::IS_ORIGIN,
                          Predictor::Action::DO_PREDICT, argReason,
                          targetOrigin, nullptr, verifier, this);
  nsAutoCString originKeyStr;
  originKey->GetAsciiSpec(originKeyStr);
  PREDICTOR_LOG(("    Predict origin=%s reason=%d action=%p", originKeyStr.get(),
                 reason, originAction.get()));
  openFlags = nsICacheStorage::OPEN_READONLY |
              nsICacheStorage::OPEN_SECRETLY |
              nsICacheStorage::CHECK_MULTITHREADED;
  mCacheDiskStorage->AsyncOpenURI(originKey,
                                  NS_LITERAL_CSTRING(PREDICTOR_ORIGIN_EXTENSION),
                                  openFlags, originAction);

  PREDICTOR_LOG(("    predict returning"));
  return NS_OK;
}

bool
Predictor::PredictInternal(PredictorPredictReason reason, nsICacheEntry *entry,
                           bool isNew, bool fullUri, nsIURI *targetURI,
                           nsINetworkPredictorVerifier *verifier,
                           uint8_t stackCount)
{
  MOZ_ASSERT(NS_IsMainThread());

  PREDICTOR_LOG(("Predictor::PredictInternal"));
  bool rv = false;

  if (reason == nsINetworkPredictor::PREDICT_LOAD) {
    MaybeLearnForStartup(targetURI, fullUri);
  }

  if (isNew) {
    // nothing else we can do here
    PREDICTOR_LOG(("    new entry"));
    return rv;
  }

  switch (reason) {
    case nsINetworkPredictor::PREDICT_LOAD:
      rv = PredictForPageload(entry, targetURI, stackCount, fullUri, verifier);
      break;
    case nsINetworkPredictor::PREDICT_STARTUP:
      rv = PredictForStartup(entry, fullUri, verifier);
      break;
    default:
      PREDICTOR_LOG(("    invalid reason"));
      MOZ_ASSERT(false, "Got unexpected value for prediction reason");
  }

  return rv;
}

void
Predictor::PredictForLink(nsIURI *targetURI, nsIURI *sourceURI,
                          nsINetworkPredictorVerifier *verifier)
{
  MOZ_ASSERT(NS_IsMainThread());

  PREDICTOR_LOG(("Predictor::PredictForLink"));
  if (!mSpeculativeService) {
    PREDICTOR_LOG(("    missing speculative service"));
    return;
  }

  if (!mEnableHoverOnSSL) {
    bool isSSL = false;
    sourceURI->SchemeIs("https", &isSSL);
    if (isSSL) {
      // We don't want to predict from an HTTPS page, to avoid info leakage
      PREDICTOR_LOG(("    Not predicting for link hover - on an SSL page"));
      return;
    }
  }

  mSpeculativeService->SpeculativeConnect2(targetURI, nullptr, nullptr);
  if (verifier) {
    PREDICTOR_LOG(("    sending verification"));
    verifier->OnPredictPreconnect(targetURI);
  }
}

// This is the driver for prediction based on a new pageload.
static const uint8_t MAX_PAGELOAD_DEPTH = 10;
bool
Predictor::PredictForPageload(nsICacheEntry *entry, nsIURI *targetURI,
                              uint8_t stackCount, bool fullUri,
                              nsINetworkPredictorVerifier *verifier)
{
  MOZ_ASSERT(NS_IsMainThread());

  PREDICTOR_LOG(("Predictor::PredictForPageload"));

  if (stackCount > MAX_PAGELOAD_DEPTH) {
    PREDICTOR_LOG(("    exceeded recursion depth!"));
    return false;
  }

  uint32_t lastLoad;
  nsresult rv = entry->GetLastFetched(&lastLoad);
  NS_ENSURE_SUCCESS(rv, false);

  int32_t globalDegradation = CalculateGlobalDegradation(lastLoad);
  PREDICTOR_LOG(("    globalDegradation = %d", globalDegradation));

  int32_t loadCount;
  rv = entry->GetFetchCount(&loadCount);
  NS_ENSURE_SUCCESS(rv, false);

  nsCOMPtr<nsIURI> redirectURI;
  if (WouldRedirect(entry, loadCount, lastLoad, globalDegradation,
                    getter_AddRefs(redirectURI))) {
    mPreconnects.AppendElement(redirectURI);
    Predictor::Reason reason;
    reason.mPredict = nsINetworkPredictor::PREDICT_LOAD;
    RefPtr<Predictor::Action> redirectAction =
      new Predictor::Action(Predictor::Action::IS_FULL_URI,
                            Predictor::Action::DO_PREDICT, reason, redirectURI,
                            nullptr, verifier, this, stackCount + 1);
    nsAutoCString redirectUriString;
    redirectURI->GetAsciiSpec(redirectUriString);
    PREDICTOR_LOG(("    Predict redirect uri=%s action=%p", redirectUriString.get(),
                   redirectAction.get()));
    uint32_t openFlags = nsICacheStorage::OPEN_READONLY |
                         nsICacheStorage::OPEN_SECRETLY |
                         nsICacheStorage::OPEN_PRIORITY |
                         nsICacheStorage::CHECK_MULTITHREADED;
    mCacheDiskStorage->AsyncOpenURI(redirectURI, EmptyCString(), openFlags,
                                    redirectAction);
    return RunPredictions(nullptr, verifier);
  }

  CalculatePredictions(entry, targetURI, lastLoad, loadCount, globalDegradation, fullUri);

  return RunPredictions(targetURI, verifier);
}

// This is the driver for predicting at browser startup time based on pages that
// have previously been loaded close to startup.
bool
Predictor::PredictForStartup(nsICacheEntry *entry, bool fullUri,
                             nsINetworkPredictorVerifier *verifier)
{
  MOZ_ASSERT(NS_IsMainThread());

  PREDICTOR_LOG(("Predictor::PredictForStartup"));
  int32_t globalDegradation = CalculateGlobalDegradation(mLastStartupTime);
  CalculatePredictions(entry, nullptr, mLastStartupTime, mStartupCount,
                       globalDegradation, fullUri);
  return RunPredictions(nullptr, verifier);
}

// This calculates how much to degrade our confidence in our data based on
// the last time this top-level resource was loaded. This "global degradation"
// applies to *all* subresources we have associated with the top-level
// resource. This will be in addition to any reduction in confidence we have
// associated with a particular subresource.
int32_t
Predictor::CalculateGlobalDegradation(uint32_t lastLoad)
{
  MOZ_ASSERT(NS_IsMainThread());

  int32_t globalDegradation;
  uint32_t delta = NOW_IN_SECONDS() - lastLoad;
  if (delta < ONE_DAY) {
    globalDegradation = mPageDegradationDay;
  } else if (delta < ONE_WEEK) {
    globalDegradation = mPageDegradationWeek;
  } else if (delta < ONE_MONTH) {
    globalDegradation = mPageDegradationMonth;
  } else if (delta < ONE_YEAR) {
    globalDegradation = mPageDegradationYear;
  } else {
    globalDegradation = mPageDegradationMax;
  }

  return globalDegradation;
}

// This calculates our overall confidence that a particular subresource will be
// loaded as part of a top-level load.
// @param hitCount - the number of times we have loaded this subresource as part
//                   of this top-level load
// @param hitsPossible - the number of times we have performed this top-level
//                       load
// @param lastHit - the timestamp of the last time we loaded this subresource as
//                  part of this top-level load
// @param lastPossible - the timestamp of the last time we performed this
//                       top-level load
// @param globalDegradation - the degradation for this top-level load as
//                            determined by CalculateGlobalDegradation
int32_t
Predictor::CalculateConfidence(uint32_t hitCount, uint32_t hitsPossible,
                               uint32_t lastHit, uint32_t lastPossible,
                               int32_t globalDegradation)
{
  MOZ_ASSERT(NS_IsMainThread());

  Telemetry::AutoCounter<Telemetry::PREDICTOR_PREDICTIONS_CALCULATED> predictionsCalculated;
  ++predictionsCalculated;

  if (!hitsPossible) {
    return 0;
  }

  int32_t baseConfidence = (hitCount * 100) / hitsPossible;
  int32_t maxConfidence = 100;
  int32_t confidenceDegradation = 0;

  if (lastHit < lastPossible) {
    // We didn't load this subresource the last time this top-level load was
    // performed, so let's not bother preconnecting (at the very least).
    maxConfidence = mPreconnectMinConfidence - 1;

    // Now calculate how much we want to degrade our confidence based on how
    // long it's been between the last time we did this top-level load and the
    // last time this top-level load included this subresource.
    PRTime delta = lastPossible - lastHit;
    if (delta == 0) {
      confidenceDegradation = 0;
    } else if (delta < ONE_DAY) {
      confidenceDegradation = mSubresourceDegradationDay;
    } else if (delta < ONE_WEEK) {
      confidenceDegradation = mSubresourceDegradationWeek;
    } else if (delta < ONE_MONTH) {
      confidenceDegradation = mSubresourceDegradationMonth;
    } else if (delta < ONE_YEAR) {
      confidenceDegradation = mSubresourceDegradationYear;
    } else {
      confidenceDegradation = mSubresourceDegradationMax;
      maxConfidence = 0;
    }
  }

  // Calculate our confidence and clamp it to between 0 and maxConfidence
  // (<= 100)
  int32_t confidence = baseConfidence - confidenceDegradation - globalDegradation;
  confidence = std::max(confidence, 0);
  confidence = std::min(confidence, maxConfidence);

  return confidence;
}

static void
MakeMetadataEntry(const uint32_t hitCount, const uint32_t lastHit,
                  const uint32_t flags, nsCString &newValue)
{
  newValue.Truncate();
  newValue.AppendInt(METADATA_VERSION);
  newValue.Append(',');
  newValue.AppendInt(hitCount);
  newValue.Append(',');
  newValue.AppendInt(lastHit);
  newValue.Append(',');
  newValue.AppendInt(flags);
}

// On every page load, the rolling window gets shifted by one bit, leaving the
// lowest bit at 0, to indicate that the subresource in question has not been
// seen on the most recent page load. If, at some point later during the page load,
// the subresource is seen again, we will then set the lowest bit to 1. This is
// how we keep track of how many of the last n pageloads (for n <= 20) a particular
// subresource has been seen.
// The rolling window is kept in the upper 20 bits of the flags element of the
// metadata. This saves 12 bits for regular old flags.
void
Predictor::UpdateRollingLoadCount(nsICacheEntry *entry, const uint32_t flags,
                                  const char *key, const uint32_t hitCount,
                                  const uint32_t lastHit)
{
  // Extract just the rolling load count from the flags, shift it to clear the
  // lowest bit, and put the new value with the existing flags.
  uint32_t rollingLoadCount = flags & ~kFlagsMask;
  rollingLoadCount <<= 1;
  uint32_t newFlags = (flags & kFlagsMask) | rollingLoadCount;

  // Finally, update the metadata on the cache entry.
  nsAutoCString newValue;
  MakeMetadataEntry(hitCount, lastHit, newFlags, newValue);
  entry->SetMetaDataElement(key, newValue.BeginReading());
}

void
Predictor::SanitizePrefs()
{
  if (mPrefetchRollingLoadCount < 0) {
    mPrefetchRollingLoadCount = 0;
  } else if (mPrefetchRollingLoadCount > kMaxPrefetchRollingLoadCount) {
    mPrefetchRollingLoadCount = kMaxPrefetchRollingLoadCount;
  }
}

void
Predictor::CalculatePredictions(nsICacheEntry *entry, nsIURI *referrer,
                                uint32_t lastLoad, uint32_t loadCount,
                                int32_t globalDegradation, bool fullUri)
{
  MOZ_ASSERT(NS_IsMainThread());

  SanitizePrefs();

  // Since the visitor gets called under a cache lock, all we do there is get
  // copies of the keys/values we care about, and then do the real work here
  entry->VisitMetaData(this);
  nsTArray<nsCString> keysToOperateOn, valuesToOperateOn;
  keysToOperateOn.SwapElements(mKeysToOperateOn);
  valuesToOperateOn.SwapElements(mValuesToOperateOn);

  MOZ_ASSERT(keysToOperateOn.Length() == valuesToOperateOn.Length());
  for (size_t i = 0; i < keysToOperateOn.Length(); ++i) {
    const char *key = keysToOperateOn[i].BeginReading();
    const char *value = valuesToOperateOn[i].BeginReading();

    nsCOMPtr<nsIURI> uri;
    uint32_t hitCount, lastHit, flags;
    if (!ParseMetaDataEntry(key, value, getter_AddRefs(uri), hitCount, lastHit, flags)) {
      // This failed, get rid of it so we don't waste space
      entry->SetMetaDataElement(key, nullptr);
      continue;
    }

    int32_t confidence = CalculateConfidence(hitCount, loadCount, lastHit,
                                             lastLoad, globalDegradation);
    if (fullUri) {
      UpdateRollingLoadCount(entry, flags, key, hitCount, lastHit);
    }
    PREDICTOR_LOG(("CalculatePredictions key=%s value=%s confidence=%d", key, value, confidence));
    if (!fullUri) {
      // Not full URI - don't prefetch! No sense in it!
      PREDICTOR_LOG(("    forcing non-cacheability - not full URI"));
      flags &= ~FLAG_PREFETCHABLE;
    } else if (!referrer) {
      // No referrer means we can't prefetch, so pretend it's non-cacheable,
      // no matter what.
      PREDICTOR_LOG(("    forcing non-cacheability - no referrer"));
      flags &= ~FLAG_PREFETCHABLE;
    } else {
      uint32_t expectedRollingLoadCount = (1 << mPrefetchRollingLoadCount) - 1;
      expectedRollingLoadCount <<= kRollingLoadOffset;
      if ((flags & expectedRollingLoadCount) != expectedRollingLoadCount) {
        PREDICTOR_LOG(("    forcing non-cacheability - missed a load"));
        flags &= ~FLAG_PREFETCHABLE;
      }
    }

    PREDICTOR_LOG(("    setting up prediction"));
    SetupPrediction(confidence, flags, uri);
  }
}

// (Maybe) adds a predictive action to the prediction runner, based on our
// calculated confidence for the subresource in question.
void
Predictor::SetupPrediction(int32_t confidence, uint32_t flags, nsIURI *uri)
{
  MOZ_ASSERT(NS_IsMainThread());

  nsAutoCString uriStr;
  uri->GetAsciiSpec(uriStr);
  PREDICTOR_LOG(("SetupPrediction mEnablePrefetch=%d mPrefetchMinConfidence=%d "
                 "mPreconnectMinConfidence=%d mPreresolveMinConfidence=%d "
                 "flags=%d confidence=%d uri=%s", mEnablePrefetch,
                 mPrefetchMinConfidence, mPreconnectMinConfidence,
                 mPreresolveMinConfidence, flags, confidence, uriStr.get()));
  if (mEnablePrefetch && (flags & FLAG_PREFETCHABLE) &&
      (mPrefetchRollingLoadCount || (confidence >= mPrefetchMinConfidence))) {
    mPrefetches.AppendElement(uri);
  } else if (confidence >= mPreconnectMinConfidence) {
    mPreconnects.AppendElement(uri);
  } else if (confidence >= mPreresolveMinConfidence) {
    mPreresolves.AppendElement(uri);
  }
}

nsresult
Predictor::Prefetch(nsIURI *uri, nsIURI *referrer,
                    nsINetworkPredictorVerifier *verifier)
{
  nsAutoCString strUri, strReferrer;
  uri->GetAsciiSpec(strUri);
  referrer->GetAsciiSpec(strReferrer);
  PREDICTOR_LOG(("Predictor::Prefetch uri=%s referrer=%s verifier=%p",
                 strUri.get(), strReferrer.get(), verifier));
  nsCOMPtr<nsIChannel> channel;
  nsresult rv = NS_NewChannel(getter_AddRefs(channel), uri,
                              nsContentUtils::GetSystemPrincipal(),
                              nsILoadInfo::SEC_ALLOW_CROSS_ORIGIN_DATA_IS_NULL,
                              nsIContentPolicy::TYPE_OTHER,
                              nullptr, /* aLoadGroup */
                              nullptr, /* aCallbacks */
                              nsIRequest::LOAD_BACKGROUND);

  if (NS_FAILED(rv)) {
    PREDICTOR_LOG(("    NS_NewChannel failed rv=0x%X", rv));
    return rv;
  }

  nsCOMPtr<nsIHttpChannel> httpChannel;
  httpChannel = do_QueryInterface(channel);
  if (!httpChannel) {
    PREDICTOR_LOG(("    Could not get HTTP Channel from new channel!"));
    return NS_ERROR_UNEXPECTED;
  }

  httpChannel->SetReferrer(referrer);
  // XXX - set a header here to indicate this is a prefetch?

  nsCOMPtr<nsIStreamListener> listener = new PrefetchListener(verifier, uri,
                                                              this);
  PREDICTOR_LOG(("    calling AsyncOpen2 listener=%p channel=%p", listener.get(),
                 channel.get()));
  rv = channel->AsyncOpen2(listener);
  if (NS_FAILED(rv)) {
    PREDICTOR_LOG(("    AsyncOpen2 failed rv=0x%X", rv));
  }

  return rv;
}

// Runs predictions that have been set up.
bool
Predictor::RunPredictions(nsIURI *referrer, nsINetworkPredictorVerifier *verifier)
{
  MOZ_ASSERT(NS_IsMainThread(), "Running prediction off main thread");

  PREDICTOR_LOG(("Predictor::RunPredictions"));

  bool predicted = false;
  uint32_t len, i;

  nsTArray<nsCOMPtr<nsIURI>> prefetches, preconnects, preresolves;
  prefetches.SwapElements(mPrefetches);
  preconnects.SwapElements(mPreconnects);
  preresolves.SwapElements(mPreresolves);

  Telemetry::AutoCounter<Telemetry::PREDICTOR_TOTAL_PREDICTIONS> totalPredictions;
  Telemetry::AutoCounter<Telemetry::PREDICTOR_TOTAL_PREFETCHES> totalPrefetches;
  Telemetry::AutoCounter<Telemetry::PREDICTOR_TOTAL_PRECONNECTS> totalPreconnects;
  Telemetry::AutoCounter<Telemetry::PREDICTOR_TOTAL_PRERESOLVES> totalPreresolves;

  len = prefetches.Length();
  for (i = 0; i < len; ++i) {
    PREDICTOR_LOG(("    doing prefetch"));
    nsCOMPtr<nsIURI> uri = prefetches[i];
    if (NS_SUCCEEDED(Prefetch(uri, referrer, verifier))) {
      ++totalPredictions;
      ++totalPrefetches;
      predicted = true;
    }
  }

  len = preconnects.Length();
  for (i = 0; i < len; ++i) {
    PREDICTOR_LOG(("    doing preconnect"));
    nsCOMPtr<nsIURI> uri = preconnects[i];
    ++totalPredictions;
    ++totalPreconnects;
    mSpeculativeService->SpeculativeConnect2(uri, nullptr, this);
    predicted = true;
    if (verifier) {
      PREDICTOR_LOG(("    sending preconnect verification"));
      verifier->OnPredictPreconnect(uri);
    }
  }

  len = preresolves.Length();
  nsCOMPtr<nsIThread> mainThread = do_GetMainThread();
  for (i = 0; i < len; ++i) {
    nsCOMPtr<nsIURI> uri = preresolves[i];
    ++totalPredictions;
    ++totalPreresolves;
    nsAutoCString hostname;
    uri->GetAsciiHost(hostname);
    PREDICTOR_LOG(("    doing preresolve %s", hostname.get()));
    nsCOMPtr<nsICancelable> tmpCancelable;
    mDnsService->AsyncResolve(hostname,
                              (nsIDNSService::RESOLVE_PRIORITY_MEDIUM |
                               nsIDNSService::RESOLVE_SPECULATE),
                              mDNSListener, nullptr,
                              getter_AddRefs(tmpCancelable));
    predicted = true;
    if (verifier) {
      PREDICTOR_LOG(("    sending preresolve verification"));
      verifier->OnPredictDNS(uri);
    }
  }

  return predicted;
}

// Find out if a top-level page is likely to redirect.
bool
Predictor::WouldRedirect(nsICacheEntry *entry, uint32_t loadCount,
                         uint32_t lastLoad, int32_t globalDegradation,
                         nsIURI **redirectURI)
{
  // TODO - not doing redirects for first go around
  MOZ_ASSERT(NS_IsMainThread());

  return false;
}

// Called from the main thread to update the database
NS_IMETHODIMP
Predictor::Learn(nsIURI *targetURI, nsIURI *sourceURI,
                 PredictorLearnReason reason,
                 nsILoadContext *loadContext)
{
  MOZ_ASSERT(NS_IsMainThread(),
             "Predictor interface methods must be called on the main thread");

  PREDICTOR_LOG(("Predictor::Learn"));

  if (IsNeckoChild()) {
    MOZ_DIAGNOSTIC_ASSERT(gNeckoChild);

    PREDICTOR_LOG(("    called on child process"));

    ipc::URIParams serTargetURI;
    SerializeURI(targetURI, serTargetURI);

    ipc::OptionalURIParams serSourceURI;
    SerializeURI(sourceURI, serSourceURI);

    IPC::SerializedLoadContext serLoadContext;
    serLoadContext.Init(loadContext);

    PREDICTOR_LOG(("    forwarding to parent"));
    gNeckoChild->SendPredLearn(serTargetURI, serSourceURI, reason,
                               serLoadContext);
    return NS_OK;
  }

  PREDICTOR_LOG(("    called on parent process"));

  if (!mInitialized) {
    PREDICTOR_LOG(("    not initialized"));
    return NS_OK;
  }

  if (!mEnabled) {
    PREDICTOR_LOG(("    not enabled"));
    return NS_OK;
  }

  if (loadContext && loadContext->UsePrivateBrowsing()) {
    // Don't want to do anything in PB mode
    PREDICTOR_LOG(("    in PB mode"));
    return NS_OK;
  }

  if (!IsNullOrHttp(targetURI) || !IsNullOrHttp(sourceURI)) {
    PREDICTOR_LOG(("    got non-HTTP[S] URI"));
    return NS_ERROR_INVALID_ARG;
  }

  nsCOMPtr<nsIURI> targetOrigin;
  nsCOMPtr<nsIURI> sourceOrigin;
  nsCOMPtr<nsIURI> uriKey;
  nsCOMPtr<nsIURI> originKey;
  nsresult rv;

  switch (reason) {
  case nsINetworkPredictor::LEARN_LOAD_TOPLEVEL:
    if (!targetURI || sourceURI) {
      PREDICTOR_LOG(("    load toplevel invalid URI state"));
      return NS_ERROR_INVALID_ARG;
    }
    rv = ExtractOrigin(targetURI, getter_AddRefs(targetOrigin), mIOService);
    NS_ENSURE_SUCCESS(rv, rv);
    uriKey = targetURI;
    originKey = targetOrigin;
    break;
  case nsINetworkPredictor::LEARN_STARTUP:
    if (!targetURI || sourceURI) {
      PREDICTOR_LOG(("    startup invalid URI state"));
      return NS_ERROR_INVALID_ARG;
    }
    rv = ExtractOrigin(targetURI, getter_AddRefs(targetOrigin), mIOService);
    NS_ENSURE_SUCCESS(rv, rv);
    uriKey = mStartupURI;
    originKey = mStartupURI;
    break;
  case nsINetworkPredictor::LEARN_LOAD_REDIRECT:
  case nsINetworkPredictor::LEARN_LOAD_SUBRESOURCE:
    if (!targetURI || !sourceURI) {
      PREDICTOR_LOG(("    redirect/subresource invalid URI state"));
      return NS_ERROR_INVALID_ARG;
    }
    rv = ExtractOrigin(targetURI, getter_AddRefs(targetOrigin), mIOService);
    NS_ENSURE_SUCCESS(rv, rv);
    rv = ExtractOrigin(sourceURI, getter_AddRefs(sourceOrigin), mIOService);
    NS_ENSURE_SUCCESS(rv, rv);
    uriKey = sourceURI;
    originKey = sourceOrigin;
    break;
  default:
    PREDICTOR_LOG(("    invalid reason"));
    return NS_ERROR_INVALID_ARG;
  }

  Telemetry::AutoCounter<Telemetry::PREDICTOR_LEARN_ATTEMPTS> learnAttempts;
  ++learnAttempts;

  Predictor::Reason argReason;
  argReason.mLearn = reason;

  // We always open the full uri (general cache) entry first, so we don't gum up
  // the works waiting on predictor-only entries to open
  RefPtr<Predictor::Action> uriAction =
    new Predictor::Action(Predictor::Action::IS_FULL_URI,
                          Predictor::Action::DO_LEARN, argReason, targetURI,
                          sourceURI, nullptr, this);
  nsAutoCString uriKeyStr, targetUriStr, sourceUriStr;
  uriKey->GetAsciiSpec(uriKeyStr);
  targetURI->GetAsciiSpec(targetUriStr);
  if (sourceURI) {
    sourceURI->GetAsciiSpec(sourceUriStr);
  }
  PREDICTOR_LOG(("    Learn uriKey=%s targetURI=%s sourceURI=%s reason=%d "
                 "action=%p", uriKeyStr.get(), targetUriStr.get(),
                 sourceUriStr.get(), reason, uriAction.get()));
  // For learning full URI things, we *always* open readonly and secretly, as we
  // rely on actual pageloads to update the entry's metadata for us.
  uint32_t uriOpenFlags = nsICacheStorage::OPEN_READONLY |
                          nsICacheStorage::OPEN_SECRETLY |
                          nsICacheStorage::CHECK_MULTITHREADED;
  if (reason == nsINetworkPredictor::LEARN_LOAD_TOPLEVEL) {
    // Learning for toplevel we want to open the full uri entry priority, since
    // it's likely this entry will be used soon anyway, and we want this to be
    // opened ASAP.
    uriOpenFlags |= nsICacheStorage::OPEN_PRIORITY;
  }
  mCacheDiskStorage->AsyncOpenURI(uriKey, EmptyCString(), uriOpenFlags,
                                  uriAction);

  // Now we open the origin-only (and therefore predictor-only) entry
  RefPtr<Predictor::Action> originAction =
    new Predictor::Action(Predictor::Action::IS_ORIGIN,
                          Predictor::Action::DO_LEARN, argReason, targetOrigin,
                          sourceOrigin, nullptr, this);
  nsAutoCString originKeyStr, targetOriginStr, sourceOriginStr;
  originKey->GetAsciiSpec(originKeyStr);
  targetOrigin->GetAsciiSpec(targetOriginStr);
  if (sourceOrigin) {
    sourceOrigin->GetAsciiSpec(sourceOriginStr);
  }
  PREDICTOR_LOG(("    Learn originKey=%s targetOrigin=%s sourceOrigin=%s reason=%d "
                 "action=%p", originKeyStr.get(), targetOriginStr.get(),
                 sourceOriginStr.get(), reason, originAction.get()));
  uint32_t originOpenFlags;
  if (reason == nsINetworkPredictor::LEARN_LOAD_TOPLEVEL) {
    // This is the only case when we want to update the 'last used' metadata on
    // the cache entry we're getting. This only applies to predictor-specific
    // entries.
    originOpenFlags = nsICacheStorage::OPEN_NORMALLY |
                      nsICacheStorage::CHECK_MULTITHREADED;
  } else {
    originOpenFlags = nsICacheStorage::OPEN_READONLY |
                      nsICacheStorage::OPEN_SECRETLY |
                      nsICacheStorage::CHECK_MULTITHREADED;
  }
  mCacheDiskStorage->AsyncOpenURI(originKey,
                                  NS_LITERAL_CSTRING(PREDICTOR_ORIGIN_EXTENSION),
                                  originOpenFlags, originAction);

  PREDICTOR_LOG(("Predictor::Learn returning"));
  return NS_OK;
}

void
Predictor::LearnInternal(PredictorLearnReason reason, nsICacheEntry *entry,
                         bool isNew, bool fullUri, nsIURI *targetURI,
                         nsIURI *sourceURI)
{
  MOZ_ASSERT(NS_IsMainThread());

  PREDICTOR_LOG(("Predictor::LearnInternal"));

  nsCString junk;
  if (!fullUri && reason == nsINetworkPredictor::LEARN_LOAD_TOPLEVEL &&
      NS_FAILED(entry->GetMetaDataElement(SEEN_META_DATA, getter_Copies(junk)))) {
    // This is an origin-only entry that we haven't seen before. Let's mark it
    // as seen.
    PREDICTOR_LOG(("    marking new origin entry as seen"));
    nsresult rv = entry->SetMetaDataElement(SEEN_META_DATA, "1");
    if (NS_FAILED(rv)) {
      PREDICTOR_LOG(("    failed to mark origin entry seen"));
      return;
    }

    // Need to ensure someone else can get to the entry if necessary
    entry->MetaDataReady();
    return;
  }

  switch (reason) {
    case nsINetworkPredictor::LEARN_LOAD_TOPLEVEL:
      // This case only exists to be used during tests - code outside the
      // predictor tests should NEVER call Learn with LEARN_LOAD_TOPLEVEL.
      // The predictor xpcshell test needs this branch, however, because we
      // have no real page loads in xpcshell, and this is how we fake it up
      // so that all the work that normally happens behind the scenes in a
      // page load can be done for testing purposes.
      if (fullUri && mDoingTests) {
        PREDICTOR_LOG(("    WARNING - updating rolling load count. "
                       "If you see this outside tests, you did it wrong"));
        SanitizePrefs();

        // Since the visitor gets called under a cache lock, all we do there is get
        // copies of the keys/values we care about, and then do the real work here
        entry->VisitMetaData(this);
        nsTArray<nsCString> keysToOperateOn, valuesToOperateOn;
        keysToOperateOn.SwapElements(mKeysToOperateOn);
        valuesToOperateOn.SwapElements(mValuesToOperateOn);

        MOZ_ASSERT(keysToOperateOn.Length() == valuesToOperateOn.Length());
        for (size_t i = 0; i < keysToOperateOn.Length(); ++i) {
          const char *key = keysToOperateOn[i].BeginReading();
          const char *value = valuesToOperateOn[i].BeginReading();

          nsCOMPtr<nsIURI> uri;
          uint32_t hitCount, lastHit, flags;
          if (!ParseMetaDataEntry(nullptr, value, nullptr, hitCount, lastHit, flags)) {
            // This failed, get rid of it so we don't waste space
            entry->SetMetaDataElement(key, nullptr);
            continue;
          }
          UpdateRollingLoadCount(entry, flags, key, hitCount, lastHit);
        }
      } else {
        PREDICTOR_LOG(("    nothing to do for toplevel"));
      }
      break;
    case nsINetworkPredictor::LEARN_LOAD_REDIRECT:
      if (fullUri) {
        LearnForRedirect(entry, targetURI);
      }
      break;
    case nsINetworkPredictor::LEARN_LOAD_SUBRESOURCE:
      LearnForSubresource(entry, targetURI);
      break;
    case nsINetworkPredictor::LEARN_STARTUP:
      LearnForStartup(entry, targetURI);
      break;
    default:
      PREDICTOR_LOG(("    unexpected reason value"));
      MOZ_ASSERT(false, "Got unexpected value for learn reason!");
  }
}

NS_IMPL_ISUPPORTS(Predictor::SpaceCleaner, nsICacheEntryMetaDataVisitor)

NS_IMETHODIMP
Predictor::SpaceCleaner::OnMetaDataElement(const char *key, const char *value)
{
  MOZ_ASSERT(NS_IsMainThread());

  if (!IsURIMetadataElement(key)) {
    // This isn't a bit of metadata we care about
    return NS_OK;
  }

  uint32_t hitCount, lastHit, flags;
  bool ok = mPredictor->ParseMetaDataEntry(nullptr, value, nullptr,
                                           hitCount, lastHit, flags);

  if (!ok) {
    // Couldn't parse this one, just get rid of it
    nsCString nsKey;
    nsKey.AssignASCII(key);
    mLongKeysToDelete.AppendElement(nsKey);
    return NS_OK;
  }

  nsCString uri(key + (sizeof(META_DATA_PREFIX) - 1));
  uint32_t uriLength = uri.Length();
  if (uriLength > mPredictor->mMaxURILength) {
    // Default to getting rid of URIs that are too long and were put in before
    // we had our limit on URI length, in order to free up some space.
    nsCString nsKey;
    nsKey.AssignASCII(key);
    mLongKeysToDelete.AppendElement(nsKey);
    return NS_OK;
  }

  if (!mLRUKeyToDelete || lastHit < mLRUStamp) {
    mLRUKeyToDelete = key;
    mLRUStamp = lastHit;
  }

  return NS_OK;
}

void
Predictor::SpaceCleaner::Finalize(nsICacheEntry *entry)
{
  MOZ_ASSERT(NS_IsMainThread());

  if (mLRUKeyToDelete) {
    entry->SetMetaDataElement(mLRUKeyToDelete, nullptr);
  }

  for (size_t i = 0; i < mLongKeysToDelete.Length(); ++i) {
    entry->SetMetaDataElement(mLongKeysToDelete[i].BeginReading(), nullptr);
  }
}

// Called when a subresource has been hit from a top-level load. Uses the two
// helper functions above to update the database appropriately.
void
Predictor::LearnForSubresource(nsICacheEntry *entry, nsIURI *targetURI)
{
  MOZ_ASSERT(NS_IsMainThread());

  PREDICTOR_LOG(("Predictor::LearnForSubresource"));

  uint32_t lastLoad;
  nsresult rv = entry->GetLastFetched(&lastLoad);
  RETURN_IF_FAILED(rv);

  int32_t loadCount;
  rv = entry->GetFetchCount(&loadCount);
  RETURN_IF_FAILED(rv);

  nsCString key;
  key.AssignLiteral(META_DATA_PREFIX);
  nsCString uri;
  targetURI->GetAsciiSpec(uri);
  key.Append(uri);
  if (uri.Length() > mMaxURILength) {
    // We do this to conserve space/prevent OOMs
    PREDICTOR_LOG(("    uri too long!"));
    entry->SetMetaDataElement(key.BeginReading(), nullptr);
    return;
  }

  nsCString value;
  rv = entry->GetMetaDataElement(key.BeginReading(), getter_Copies(value));

  uint32_t hitCount, lastHit, flags;
  bool isNewResource = (NS_FAILED(rv) ||
                        !ParseMetaDataEntry(nullptr, value.BeginReading(),
                                            nullptr, hitCount, lastHit, flags));

  int32_t resourceCount = 0;
  if (isNewResource) {
    // This is a new addition
    PREDICTOR_LOG(("    new resource"));
    nsCString s;
    rv = entry->GetMetaDataElement(RESOURCE_META_DATA, getter_Copies(s));
    if (NS_SUCCEEDED(rv)) {
      resourceCount = atoi(s.BeginReading());
    }
    if (resourceCount >= mMaxResourcesPerEntry) {
      RefPtr<Predictor::SpaceCleaner> cleaner =
        new Predictor::SpaceCleaner(this);
      entry->VisitMetaData(cleaner);
      cleaner->Finalize(entry);
    } else {
      ++resourceCount;
    }
    nsAutoCString count;
    count.AppendInt(resourceCount);
    rv = entry->SetMetaDataElement(RESOURCE_META_DATA, count.BeginReading());
    if (NS_FAILED(rv)) {
      PREDICTOR_LOG(("    failed to update resource count"));
      return;
    }
    hitCount = 1;
    flags = 0;
  } else {
    PREDICTOR_LOG(("    existing resource"));
    hitCount = std::min(hitCount + 1, static_cast<uint32_t>(loadCount));
  }

  // Update the rolling load count to mark this sub-resource as seen on the
  // most-recent pageload so it can be eligible for prefetch (assuming all
  // the other stars align).
  flags |= (1 << kRollingLoadOffset);

  nsCString newValue;
  MakeMetadataEntry(hitCount, lastLoad, flags, newValue);
  rv = entry->SetMetaDataElement(key.BeginReading(), newValue.BeginReading());
  PREDICTOR_LOG(("    SetMetaDataElement -> 0x%08X", rv));
  if (NS_FAILED(rv) && isNewResource) {
    // Roll back the increment to the resource count we made above.
    PREDICTOR_LOG(("    rolling back resource count update"));
    --resourceCount;
    if (resourceCount == 0) {
      entry->SetMetaDataElement(RESOURCE_META_DATA, nullptr);
    } else {
      nsAutoCString count;
      count.AppendInt(resourceCount);
      entry->SetMetaDataElement(RESOURCE_META_DATA, count.BeginReading());
    }
  }
}

// This is called when a top-level loaded ended up redirecting to a different
// URI so we can keep track of that fact.
void
Predictor::LearnForRedirect(nsICacheEntry *entry, nsIURI *targetURI)
{
  MOZ_ASSERT(NS_IsMainThread());

  // TODO - not doing redirects for first go around
  PREDICTOR_LOG(("Predictor::LearnForRedirect"));
}

// This will add a page to our list of startup pages if it's being loaded
// before our startup window has expired.
void
Predictor::MaybeLearnForStartup(nsIURI *uri, bool fullUri)
{
  MOZ_ASSERT(NS_IsMainThread());

  // TODO - not doing startup for first go around
  PREDICTOR_LOG(("Predictor::MaybeLearnForStartup"));
}

// Add information about a top-level load to our list of startup pages
void
Predictor::LearnForStartup(nsICacheEntry *entry, nsIURI *targetURI)
{
  MOZ_ASSERT(NS_IsMainThread());

  // These actually do the same set of work, just on different entries, so we
  // can pass through to get the real work done here
  PREDICTOR_LOG(("Predictor::LearnForStartup"));
  LearnForSubresource(entry, targetURI);
}

bool
Predictor::ParseMetaDataEntry(const char *key, const char *value, nsIURI **uri,
                              uint32_t &hitCount, uint32_t &lastHit,
                              uint32_t &flags)
{
  MOZ_ASSERT(NS_IsMainThread());

  PREDICTOR_LOG(("Predictor::ParseMetaDataEntry key=%s value=%s",
                 key ? key : "", value));

  const char *comma = strchr(value, ',');
  if (!comma) {
    PREDICTOR_LOG(("    could not find first comma"));
    return false;
  }

  uint32_t version = static_cast<uint32_t>(atoi(value));
  PREDICTOR_LOG(("    version -> %u", version));

  if (version != METADATA_VERSION) {
    PREDICTOR_LOG(("    metadata version mismatch %u != %u", version,
                   METADATA_VERSION));
    return false;
  }

  value = comma + 1;
  comma = strchr(value, ',');
  if (!comma) {
    PREDICTOR_LOG(("    could not find second comma"));
    return false;
  }

  hitCount = static_cast<uint32_t>(atoi(value));
  PREDICTOR_LOG(("    hitCount -> %u", hitCount));

  value = comma + 1;
  comma = strchr(value, ',');
  if (!comma) {
    PREDICTOR_LOG(("    could not find third comma"));
    return false;
  }

  lastHit = static_cast<uint32_t>(atoi(value));
  PREDICTOR_LOG(("    lastHit -> %u", lastHit));

  value = comma + 1;
  flags = static_cast<uint32_t>(atoi(value));
  PREDICTOR_LOG(("    flags -> %u", flags));

  if (key) {
    const char *uriStart = key + (sizeof(META_DATA_PREFIX) - 1);
    nsresult rv = NS_NewURI(uri, uriStart, nullptr, mIOService);
    if (NS_FAILED(rv)) {
      PREDICTOR_LOG(("    NS_NewURI returned 0x%X", rv));
      return false;
    }
    PREDICTOR_LOG(("    uri -> %s", uriStart));
  }

  return true;
}

NS_IMETHODIMP
Predictor::Reset()
{
  MOZ_ASSERT(NS_IsMainThread(),
             "Predictor interface methods must be called on the main thread");

  PREDICTOR_LOG(("Predictor::Reset"));

  if (IsNeckoChild()) {
    MOZ_DIAGNOSTIC_ASSERT(gNeckoChild);

    PREDICTOR_LOG(("    forwarding to parent process"));
    gNeckoChild->SendPredReset();
    return NS_OK;
  }

  PREDICTOR_LOG(("    called on parent process"));

  if (!mInitialized) {
    PREDICTOR_LOG(("    not initialized"));
    return NS_OK;
  }

  if (!mEnabled) {
    PREDICTOR_LOG(("    not enabled"));
    return NS_OK;
  }

  RefPtr<Predictor::Resetter> reset = new Predictor::Resetter(this);
  PREDICTOR_LOG(("    created a resetter"));
  mCacheDiskStorage->AsyncVisitStorage(reset, true);
  PREDICTOR_LOG(("    Cache async launched, returning now"));

  return NS_OK;
}

NS_IMPL_ISUPPORTS(Predictor::Resetter,
                  nsICacheEntryOpenCallback,
                  nsICacheEntryMetaDataVisitor,
                  nsICacheStorageVisitor);

Predictor::Resetter::Resetter(Predictor *predictor)
  :mEntriesToVisit(0)
  ,mPredictor(predictor)
{ }

NS_IMETHODIMP
Predictor::Resetter::OnCacheEntryCheck(nsICacheEntry *entry,
                                       nsIApplicationCache *appCache,
                                       uint32_t *result)
{
  *result = nsICacheEntryOpenCallback::ENTRY_WANTED;
  return NS_OK;
}

NS_IMETHODIMP
Predictor::Resetter::OnCacheEntryAvailable(nsICacheEntry *entry, bool isNew,
                                           nsIApplicationCache *appCache,
                                           nsresult result)
{
  MOZ_ASSERT(NS_IsMainThread());

  if (NS_FAILED(result)) {
    // This can happen when we've tried to open an entry that doesn't exist for
    // some non-reset operation, and then get reset shortly thereafter (as
    // happens in some of our tests).
    --mEntriesToVisit;
    if (!mEntriesToVisit) {
      Complete();
    }
    return NS_OK;
  }

  entry->VisitMetaData(this);
  nsTArray<nsCString> keysToDelete;
  keysToDelete.SwapElements(mKeysToDelete);

  for (size_t i = 0; i < keysToDelete.Length(); ++i) {
    const char *key = keysToDelete[i].BeginReading();
    entry->SetMetaDataElement(key, nullptr);
  }

  --mEntriesToVisit;
  if (!mEntriesToVisit) {
    Complete();
  }

  return NS_OK;
}

NS_IMETHODIMP
Predictor::Resetter::OnMetaDataElement(const char *asciiKey,
                                       const char *asciiValue)
{
  MOZ_ASSERT(NS_IsMainThread());

  if (!StringBeginsWith(nsDependentCString(asciiKey),
                        NS_LITERAL_CSTRING(META_DATA_PREFIX))) {
    // Not a metadata entry we care about, carry on
    return NS_OK;
  }

  nsCString key;
  key.AssignASCII(asciiKey);
  mKeysToDelete.AppendElement(key);

  return NS_OK;
}

NS_IMETHODIMP
Predictor::Resetter::OnCacheStorageInfo(uint32_t entryCount, uint64_t consumption,
                                        uint64_t capacity, nsIFile *diskDirectory)
{
  MOZ_ASSERT(NS_IsMainThread());

  return NS_OK;
}

NS_IMETHODIMP
Predictor::Resetter::OnCacheEntryInfo(nsIURI *uri, const nsACString &idEnhance,
                                      int64_t dataSize, int32_t fetchCount,
                                      uint32_t lastModifiedTime, uint32_t expirationTime,
                                      bool aPinned)
{
  MOZ_ASSERT(NS_IsMainThread());

  // The predictor will only ever touch entries with no idEnhance ("") or an
  // idEnhance of PREDICTOR_ORIGIN_EXTENSION, so we filter out any entries that
  // don't match that to avoid doing extra work.
  if (idEnhance.EqualsLiteral(PREDICTOR_ORIGIN_EXTENSION)) {
    // This is an entry we own, so we can just doom it entirely
    mPredictor->mCacheDiskStorage->AsyncDoomURI(uri, idEnhance, nullptr);
  } else if (idEnhance.IsEmpty()) {
    // This is an entry we don't own, so we have to be a little more careful and
    // just get rid of our own metadata entries. Append it to an array of things
    // to operate on and then do the operations later so we don't end up calling
    // Complete() multiple times/too soon.
    ++mEntriesToVisit;
    mURIsToVisit.AppendElement(uri);
  }

  return NS_OK;
}

NS_IMETHODIMP
Predictor::Resetter::OnCacheEntryVisitCompleted()
{
  MOZ_ASSERT(NS_IsMainThread());

  nsTArray<nsCOMPtr<nsIURI>> urisToVisit;
  urisToVisit.SwapElements(mURIsToVisit);

  MOZ_ASSERT(mEntriesToVisit == urisToVisit.Length());
  if (!mEntriesToVisit) {
    Complete();
    return NS_OK;
  }

  uint32_t entriesToVisit = urisToVisit.Length();
  for (uint32_t i = 0; i < entriesToVisit; ++i) {
    nsCString u;
    urisToVisit[i]->GetAsciiSpec(u);
    mPredictor->mCacheDiskStorage->AsyncOpenURI(
        urisToVisit[i], EmptyCString(),
        nsICacheStorage::OPEN_READONLY | nsICacheStorage::OPEN_SECRETLY | nsICacheStorage::CHECK_MULTITHREADED,
        this);
  }

  return NS_OK;
}

void
Predictor::Resetter::Complete()
{
  MOZ_ASSERT(NS_IsMainThread());

  nsCOMPtr<nsIObserverService> obs = mozilla::services::GetObserverService();
  if (!obs) {
    PREDICTOR_LOG(("COULD NOT GET OBSERVER SERVICE!"));
    return;
  }

  obs->NotifyObservers(nullptr, "predictor-reset-complete", nullptr);
}

// Helper functions to make using the predictor easier from native code

static nsresult
EnsureGlobalPredictor(nsINetworkPredictor **aPredictor)
{
  MOZ_ASSERT(NS_IsMainThread());

  nsresult rv;
  nsCOMPtr<nsINetworkPredictor> predictor =
    do_GetService("@mozilla.org/network/predictor;1",
                  &rv);
  NS_ENSURE_SUCCESS(rv, rv);

  predictor.forget(aPredictor);
  return NS_OK;
}

nsresult
PredictorPredict(nsIURI *targetURI, nsIURI *sourceURI,
                 PredictorPredictReason reason, nsILoadContext *loadContext,
                 nsINetworkPredictorVerifier *verifier)
{
  MOZ_ASSERT(NS_IsMainThread());

  if (!IsNullOrHttp(targetURI) || !IsNullOrHttp(sourceURI)) {
    return NS_OK;
  }

  nsCOMPtr<nsINetworkPredictor> predictor;
  nsresult rv = EnsureGlobalPredictor(getter_AddRefs(predictor));
  NS_ENSURE_SUCCESS(rv, rv);

  return predictor->Predict(targetURI, sourceURI, reason,
                            loadContext, verifier);
}

nsresult
PredictorLearn(nsIURI *targetURI, nsIURI *sourceURI,
               PredictorLearnReason reason,
               nsILoadContext *loadContext)
{
  MOZ_ASSERT(NS_IsMainThread());

  if (!IsNullOrHttp(targetURI) || !IsNullOrHttp(sourceURI)) {
    return NS_OK;
  }

  nsCOMPtr<nsINetworkPredictor> predictor;
  nsresult rv = EnsureGlobalPredictor(getter_AddRefs(predictor));
  NS_ENSURE_SUCCESS(rv, rv);

  return predictor->Learn(targetURI, sourceURI, reason, loadContext);
}

nsresult
PredictorLearn(nsIURI *targetURI, nsIURI *sourceURI,
               PredictorLearnReason reason,
               nsILoadGroup *loadGroup)
{
  MOZ_ASSERT(NS_IsMainThread());

  if (!IsNullOrHttp(targetURI) || !IsNullOrHttp(sourceURI)) {
    return NS_OK;
  }

  nsCOMPtr<nsINetworkPredictor> predictor;
  nsresult rv = EnsureGlobalPredictor(getter_AddRefs(predictor));
  NS_ENSURE_SUCCESS(rv, rv);

  nsCOMPtr<nsILoadContext> loadContext;

  if (loadGroup) {
    nsCOMPtr<nsIInterfaceRequestor> callbacks;
    loadGroup->GetNotificationCallbacks(getter_AddRefs(callbacks));
    if (callbacks) {
      loadContext = do_GetInterface(callbacks);
    }
  }

  return predictor->Learn(targetURI, sourceURI, reason, loadContext);
}

nsresult
PredictorLearn(nsIURI *targetURI, nsIURI *sourceURI,
               PredictorLearnReason reason,
               nsIDocument *document)
{
  MOZ_ASSERT(NS_IsMainThread());

  if (!IsNullOrHttp(targetURI) || !IsNullOrHttp(sourceURI)) {
    return NS_OK;
  }

  nsCOMPtr<nsINetworkPredictor> predictor;
  nsresult rv = EnsureGlobalPredictor(getter_AddRefs(predictor));
  NS_ENSURE_SUCCESS(rv, rv);

  nsCOMPtr<nsILoadContext> loadContext;

  if (document) {
    loadContext = document->GetLoadContext();
  }

  return predictor->Learn(targetURI, sourceURI, reason, loadContext);
}

nsresult
PredictorLearnRedirect(nsIURI *targetURI, nsIChannel *channel,
                       nsILoadContext *loadContext)
{
  MOZ_ASSERT(NS_IsMainThread());

  nsCOMPtr<nsIURI> sourceURI;
  nsresult rv = channel->GetOriginalURI(getter_AddRefs(sourceURI));
  NS_ENSURE_SUCCESS(rv, rv);

  bool sameUri;
  rv = targetURI->Equals(sourceURI, &sameUri);
  NS_ENSURE_SUCCESS(rv, rv);

  if (sameUri) {
    return NS_OK;
  }

  if (!IsNullOrHttp(targetURI) || !IsNullOrHttp(sourceURI)) {
    return NS_OK;
  }

  nsCOMPtr<nsINetworkPredictor> predictor;
  rv = EnsureGlobalPredictor(getter_AddRefs(predictor));
  NS_ENSURE_SUCCESS(rv, rv);

  return predictor->Learn(targetURI, sourceURI,
                          nsINetworkPredictor::LEARN_LOAD_REDIRECT,
                          loadContext);
}

// nsINetworkPredictorVerifier

/**
 * Call through to the child's verifier (only during tests)
 */
NS_IMETHODIMP
Predictor::OnPredictPrefetch(nsIURI *aURI, uint32_t httpStatus)
{
  if (IsNeckoChild()) {
    if (mChildVerifier) {
      // Ideally, we'd assert here. But since we're slowly moving towards a
      // world where we have multiple child processes, and only one child process
      // will be likely to have a verifier, we have to play it safer.
      return mChildVerifier->OnPredictPrefetch(aURI, httpStatus);
    }
    return NS_OK;
  }

  ipc::URIParams serURI;
  SerializeURI(aURI, serURI);

  for (auto* cp : ContentParent::AllProcesses(ContentParent::eLive)) {
    PNeckoParent* neckoParent = SingleManagedOrNull(cp->ManagedPNeckoParent());
    if (!neckoParent) {
      continue;
    }
    if (!neckoParent->SendPredOnPredictPrefetch(serURI, httpStatus)) {
      return NS_ERROR_NOT_AVAILABLE;
    }
  }

  return NS_OK;
}

NS_IMETHODIMP
Predictor::OnPredictPreconnect(nsIURI *aURI) {
  if (IsNeckoChild()) {
    if (mChildVerifier) {
      // Ideally, we'd assert here. But since we're slowly moving towards a
      // world where we have multiple child processes, and only one child process
      // will be likely to have a verifier, we have to play it safer.
      return mChildVerifier->OnPredictPreconnect(aURI);
    }
    return NS_OK;
  }

  ipc::URIParams serURI;
  SerializeURI(aURI, serURI);

  for (auto* cp : ContentParent::AllProcesses(ContentParent::eLive)) {
    PNeckoParent* neckoParent = SingleManagedOrNull(cp->ManagedPNeckoParent());
    if (!neckoParent) {
      continue;
    }
    if (!neckoParent->SendPredOnPredictPreconnect(serURI)) {
      return NS_ERROR_NOT_AVAILABLE;
    }
  }

  return NS_OK;
}

NS_IMETHODIMP
Predictor::OnPredictDNS(nsIURI *aURI) {
  if (IsNeckoChild()) {
    if (mChildVerifier) {
      // Ideally, we'd assert here. But since we're slowly moving towards a
      // world where we have multiple child processes, and only one child process
      // will be likely to have a verifier, we have to play it safer.
      return mChildVerifier->OnPredictDNS(aURI);
    }
    return NS_OK;
  }

  ipc::URIParams serURI;
  SerializeURI(aURI, serURI);

  for (auto* cp : ContentParent::AllProcesses(ContentParent::eLive)) {
    PNeckoParent* neckoParent = SingleManagedOrNull(cp->ManagedPNeckoParent());
    if (!neckoParent) {
      continue;
    }
    if (!neckoParent->SendPredOnPredictDNS(serURI)) {
      return NS_ERROR_NOT_AVAILABLE;
    }
  }

  return NS_OK;
}

// Predictor::PrefetchListener
// nsISupports
NS_IMPL_ISUPPORTS(Predictor::PrefetchListener,
                  nsIStreamListener,
                  nsIRequestObserver)

// nsIRequestObserver
NS_IMETHODIMP
Predictor::PrefetchListener::OnStartRequest(nsIRequest *aRequest,
                                            nsISupports *aContext)
{
  mStartTime = TimeStamp::Now();
  return NS_OK;
}

NS_IMETHODIMP
Predictor::PrefetchListener::OnStopRequest(nsIRequest *aRequest,
                                           nsISupports *aContext,
                                           nsresult aStatusCode)
{
  PREDICTOR_LOG(("OnStopRequest this=%p aStatusCode=0x%X", this, aStatusCode));
  NS_ENSURE_ARG(aRequest);
  if (NS_FAILED(aStatusCode)) {
    return aStatusCode;
  }

  nsCOMPtr<nsIHttpChannel> httpChannel = do_QueryInterface(aRequest);
  if (!httpChannel) {
    PREDICTOR_LOG(("    Could not get HTTP Channel!"));
    return NS_ERROR_UNEXPECTED;
  }
  nsCOMPtr<nsICachingChannel> cachingChannel = do_QueryInterface(httpChannel);
  if (!cachingChannel) {
    PREDICTOR_LOG(("    Could not get caching channel!"));
    return NS_ERROR_UNEXPECTED;
  }

  nsresult rv = NS_OK;
  uint32_t httpStatus;
  rv = httpChannel->GetResponseStatus(&httpStatus);
  if (NS_SUCCEEDED(rv) && httpStatus == 200) {
    rv = cachingChannel->ForceCacheEntryValidFor(mPredictor->mPrefetchForceValidFor);
    PREDICTOR_LOG(("    forcing entry valid for %d seconds rv=%X",
                   mPredictor->mPrefetchForceValidFor, rv));
  } else {
    rv = cachingChannel->ForceCacheEntryValidFor(0);
    PREDICTOR_LOG(("    removing any forced validity rv=%X", rv));
  }

  nsAutoCString reqName;
  rv = aRequest->GetName(reqName);
  if (NS_FAILED(rv)) {
    reqName.AssignLiteral("<unknown>");
  }

  PREDICTOR_LOG(("    request %s status %u", reqName.get(), httpStatus));

  if (mVerifier) {
    mVerifier->OnPredictPrefetch(mURI, httpStatus);
  }

  return rv;
}

// nsIStreamListener
NS_IMETHODIMP
Predictor::PrefetchListener::OnDataAvailable(nsIRequest *aRequest,
                                             nsISupports *aContext,
                                             nsIInputStream *aInputStream,
                                             uint64_t aOffset,
                                             const uint32_t aCount)
{
  uint32_t result;
  return aInputStream->ReadSegments(NS_DiscardSegment, nullptr, aCount, &result);
}

// Miscellaneous Predictor

void
Predictor::UpdateCacheability(nsIURI *sourceURI, nsIURI *targetURI,
                              uint32_t httpStatus,
                              nsHttpRequestHead &requestHead,
                              nsHttpResponseHead *responseHead,
                              nsILoadContextInfo *lci)
{
  MOZ_ASSERT(NS_IsMainThread());

  if (lci && lci->IsPrivate()) {
    PREDICTOR_LOG(("Predictor::UpdateCacheability in PB mode - ignoring"));
    return;
  }

  if (!sourceURI || !targetURI) {
    PREDICTOR_LOG(("Predictor::UpdateCacheability missing source or target uri"));
    return;
  }

  if (!IsNullOrHttp(sourceURI) || !IsNullOrHttp(targetURI)) {
    PREDICTOR_LOG(("Predictor::UpdateCacheability non-http(s) uri"));
    return;
  }

  RefPtr<Predictor> self = sSelf;
  if (self) {
    nsAutoCString method;
    requestHead.Method(method);
    self->UpdateCacheabilityInternal(sourceURI, targetURI, httpStatus,
                                     method);
  }
}

void
Predictor::UpdateCacheabilityInternal(nsIURI *sourceURI, nsIURI *targetURI,
                                      uint32_t httpStatus,
                                      const nsCString &method)
{
  PREDICTOR_LOG(("Predictor::UpdateCacheability httpStatus=%u", httpStatus));

  if (!mInitialized) {
    PREDICTOR_LOG(("    not initialized"));
    return;
  }

  if (!mEnabled) {
    PREDICTOR_LOG(("    not enabled"));
    return;
  }

  if (!mEnablePrefetch) {
    PREDICTOR_LOG(("    prefetch not enabled"));
    return;
  }

  uint32_t openFlags = nsICacheStorage::OPEN_READONLY |
                       nsICacheStorage::OPEN_SECRETLY |
                       nsICacheStorage::CHECK_MULTITHREADED;
  RefPtr<Predictor::CacheabilityAction> action =
    new Predictor::CacheabilityAction(targetURI, httpStatus, method, this);
  nsAutoCString uri;
  targetURI->GetAsciiSpec(uri);
  PREDICTOR_LOG(("    uri=%s action=%p", uri.get(), action.get()));
  mCacheDiskStorage->AsyncOpenURI(sourceURI, EmptyCString(), openFlags, action);
}

NS_IMPL_ISUPPORTS(Predictor::CacheabilityAction,
                  nsICacheEntryOpenCallback,
                  nsICacheEntryMetaDataVisitor);

NS_IMETHODIMP
Predictor::CacheabilityAction::OnCacheEntryCheck(nsICacheEntry *entry,
                                                 nsIApplicationCache *appCache,
                                                 uint32_t *result)
{
  *result = nsICacheEntryOpenCallback::ENTRY_WANTED;
  return NS_OK;
}

NS_IMETHODIMP
Predictor::CacheabilityAction::OnCacheEntryAvailable(nsICacheEntry *entry,
                                                     bool isNew,
                                                     nsIApplicationCache *appCache,
                                                     nsresult result)
{
  MOZ_ASSERT(NS_IsMainThread());
  // This is being opened read-only, so isNew should always be false
  MOZ_ASSERT(!isNew);

  PREDICTOR_LOG(("CacheabilityAction::OnCacheEntryAvailable this=%p", this));
  if (NS_FAILED(result)) {
    // Nothing to do
    PREDICTOR_LOG(("    nothing to do result=%X isNew=%d", result, isNew));
    return NS_OK;
  }

  nsresult rv = entry->VisitMetaData(this);
  if (NS_FAILED(rv)) {
    PREDICTOR_LOG(("    VisitMetaData returned %x", rv));
    return NS_OK;
  }

  nsTArray<nsCString> keysToCheck, valuesToCheck;
  keysToCheck.SwapElements(mKeysToCheck);
  valuesToCheck.SwapElements(mValuesToCheck);

  MOZ_ASSERT(keysToCheck.Length() == valuesToCheck.Length());
  for (size_t i = 0; i < keysToCheck.Length(); ++i) {
    const char *key = keysToCheck[i].BeginReading();
    const char *value = valuesToCheck[i].BeginReading();
    nsCOMPtr<nsIURI> uri;
    uint32_t hitCount, lastHit, flags;

    if (!mPredictor->ParseMetaDataEntry(key, value, getter_AddRefs(uri),
                                        hitCount, lastHit, flags)) {
      PREDICTOR_LOG(("    failed to parse key=%s value=%s", key, value));
      continue;
    }

    bool eq = false;
    if (NS_SUCCEEDED(uri->Equals(mTargetURI, &eq)) && eq) {
      if (mHttpStatus == 200 && mMethod.EqualsLiteral("GET")) {
        PREDICTOR_LOG(("    marking %s cacheable", key));
        flags |= FLAG_PREFETCHABLE;
      } else {
        PREDICTOR_LOG(("    marking %s uncacheable", key));
        flags &= ~FLAG_PREFETCHABLE;
      }
      nsCString newValue;
      MakeMetadataEntry(hitCount, lastHit, flags, newValue);
      entry->SetMetaDataElement(key, newValue.BeginReading());
      break;
    }
  }

  return NS_OK;
}

NS_IMETHODIMP
Predictor::CacheabilityAction::OnMetaDataElement(const char *asciiKey,
                                                 const char *asciiValue)
{
  MOZ_ASSERT(NS_IsMainThread());

  if (!IsURIMetadataElement(asciiKey)) {
    return NS_OK;
  }

  nsCString key, value;
  key.AssignASCII(asciiKey);
  value.AssignASCII(asciiValue);
  mKeysToCheck.AppendElement(key);
  mValuesToCheck.AppendElement(value);

  return NS_OK;
}

} // namespace net
} // namespace mozilla