ematrix/js/storage.js

/*******************************************************************************

    ηMatrix - a browser extension to black/white list requests.
    Copyright (C) 2014-2019 Raymond Hill
    Copyright (C) 2019-2022 Alessio Vanni

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program.  If not, see {http://www.gnu.org/licenses/}.

    Home: https://gitlab.com/vannilla/ematrix
    uMatrix Home: https://github.com/gorhill/uMatrix
*/

/* global objectAssign, publicSuffixList */

'use strict';

Components.utils.import('chrome://ematrix/content/lib/PublicSuffixList.jsm');
Components.utils.import('chrome://ematrix/content/lib/Tools.jsm');

ηMatrix.getBytesInUse = function () {
    let ηm = this;
    let getBytesInUseHandler = function (bytesInUse) {
        ηm.storageUsed = bytesInUse;
    };

    // Not all WebExtension implementations support getBytesInUse().
    // ηMatrix: not really our business, but does it impact us?
    if (typeof vAPI.storage.getBytesInUse === 'function') {
        vAPI.storage.getBytesInUse(null, getBytesInUseHandler);
    } else {
        ηm.storageUsed = undefined;
    }
};

ηMatrix.saveUserSettings = function () {
    this.XAL.keyvalSetMany(this.userSettings,
			   this.getBytesInUse.bind(this));
};

ηMatrix.loadUserSettings = function (callback) {
    let ηm = this;

    if (typeof callback !== 'function') {
        callback = this.noopFunc;
    }

    var settingsLoaded = function (store) {
        // console.log('storage.js > loaded user settings');
        ηm.userSettings = store;
        callback(ηm.userSettings);
    };

    vAPI.storage.get(this.userSettings, settingsLoaded);
};

ηMatrix.loadRawSettings = function () {
    let ηm = this;
    let onLoaded = function (bin) {
        if (!bin || bin.rawSettings instanceof Object === false) {
	    return;
	}

        for (let key of Object.keys(bin.rawSettings)) {
            if (ηm.rawSettings.hasOwnProperty(key) === false
		|| typeof bin.rawSettings[key] !== typeof ηm.rawSettings[key]) {
                continue;
            }
            ηm.rawSettings[key] = bin.rawSettings[key];
        }
        ηm.rawSettingsWriteTime = Date.now();
    };

    vAPI.storage.get('rawSettings', onLoaded);
};

ηMatrix.saveRawSettings = function (rawSettings, callback) {
    let keys = Object.keys(rawSettings);
    if (keys.length === 0) {
        if (typeof callback === 'function') {
            callback();
        }
        return;
    }

    for (let key of keys) {
        if (this.rawSettingsDefault.hasOwnProperty(key)
	    && typeof rawSettings[key] === typeof this.rawSettingsDefault[key]) {
            this.rawSettings[key] = rawSettings[key];
        }
    }
    vAPI.storage.set({
	rawSettings: this.rawSettings
    }, callback);
    this.rawSettingsWriteTime = Date.now();
};

ηMatrix.rawSettingsFromString = function (raw) {
    let result = {};
    let lineIter = new Tools.LineIterator(raw);

    while (lineIter.eot() === false) {
        let line = lineIter.next().trim();
        let matches = /^(\S+)(\s+(.+))?$/.exec(line);

        if (matches === null) {
	    continue;
	}

        let name = matches[1];
        if (this.rawSettingsDefault.hasOwnProperty(name) === false) {
            continue;
        }

        let value = (matches[2] || '').trim();

        switch (typeof this.rawSettingsDefault[name]) {
        case 'boolean':
            if (value === 'true') {
                value = true;
            } else if (value === 'false') {
                value = false;
            } else {
                value = this.rawSettingsDefault[name];
            }
            break;
        case 'string':
            if (value === '') {
                value = this.rawSettingsDefault[name];
            }
            break;
        case 'number':
            value = parseInt(value, 10);
            if (isNaN(value)) {
                value = this.rawSettingsDefault[name];
            }
            break;
        default:
            break;
        }
        if (this.rawSettings[name] !== value) {
            result[name] = value;
        }
    }
    this.saveRawSettings(result);
};

ηMatrix.stringFromRawSettings = function () {
    let out = [];
    for (let key of Object.keys(this.rawSettings).sort()) {
        out.push(key + ' ' + this.rawSettings[key]);
    }
    return out.join('\n');
};

// save white/blacklist
ηMatrix.saveMatrix = function () {
    ηMatrix.XAL.keyvalSetOne('userMatrix', this.pMatrix.toString());
};

ηMatrix.loadMatrix = function (callback) {
    if (typeof callback !== 'function') {
        callback = this.noopFunc;
    }

    let ηm = this;
    let onLoaded = function (bin) {
        if (bin.hasOwnProperty('userMatrix')) {
            ηm.pMatrix.fromString(bin.userMatrix);
            ηm.tMatrix.assign(ηm.pMatrix);
            callback();
        }
    };

    this.XAL.keyvalGetOne('userMatrix', onLoaded);
};

ηMatrix.listKeysFromCustomHostsFiles = function (raw) {
    let out = new Set();
    let reIgnore = /^[!#]/;
    let reValid = /^[a-z-]+:\/\/\S+/;
    let lineIter = new Tools.LineIterator(raw);

    while (lineIter.eot() === false) {
        let location = lineIter.next().trim();
        if (reIgnore.test(location) || !reValid.test(location)) {
	    continue;
	}
        out.add(location);
    }
    return Tools.setToArray(out);
};

ηMatrix.getAvailableHostsFiles = function (callback) {
    let ηm = this;
    let availableHostsFiles = {};

    // Custom filter lists.
    let importedListKeys =
	this.listKeysFromCustomHostsFiles(ηm.userSettings.externalHostsFiles);
    let i = importedListKeys.length;

    while (i--) {
        let listKey = importedListKeys[i];
        let entry = {
            content: 'filters',
            contentURL: listKey,
            external: true,
            submitter: 'user',
            title: listKey
        };
        availableHostsFiles[listKey] = entry;
        this.assets.registerAssetSource(listKey, entry);
    }

    // selected lists
    let onSelectedHostsFilesLoaded = function (bin) {
        // Now get user's selection of lists
        for (let assetKey in bin.liveHostsFiles) {
            let availableEntry = availableHostsFiles[assetKey];
            if (availableEntry === undefined) {
		continue;
	    }

            let liveEntry = bin.liveHostsFiles[assetKey];
            availableEntry.off = liveEntry.off || false;
            if (liveEntry.entryCount !== undefined) {
                availableEntry.entryCount = liveEntry.entryCount;
            }
            if (liveEntry.entryUsedCount !== undefined) {
                availableEntry.entryUsedCount = liveEntry.entryUsedCount;
            }

            // This may happen if the list name was pulled from the list content
            if (availableEntry.title === '' && liveEntry.title !== undefined) {
                availableEntry.title = liveEntry.title;
            }
        }

        // Remove unreferenced imported filter lists.
        let dict = new Set(importedListKeys);
        for (let assetKey in availableHostsFiles) {
            let entry = availableHostsFiles[assetKey];
            if (entry.submitter !== 'user') {
		continue;
	    }

            if (dict.has(assetKey)) {
		continue;
	    }

            delete availableHostsFiles[assetKey];
            ηm.assets.unregisterAssetSource(assetKey);
            ηm.assets.remove(assetKey);
        }

        callback(availableHostsFiles);
    };

    // built-in lists
    let onBuiltinHostsFilesLoaded = function (entries) {
        for (let assetKey in entries) {
            if (entries.hasOwnProperty(assetKey) === false) {
		continue;
	    }

            let entry = entries[assetKey];
            if (entry.content !== 'filters') {
		continue;
	    }
            availableHostsFiles[assetKey] = Object.assign({}, entry);
        }

        // Now get user's selection of lists
        vAPI.storage.get({
	    'liveHostsFiles': availableHostsFiles
	}, onSelectedHostsFilesLoaded);
    };

    this.assets.metadata(onBuiltinHostsFilesLoaded);
};

ηMatrix.loadHostsFiles = function (callback) {
    let ηm = ηMatrix;
    let hostsFileLoadCount;

    if (typeof callback !== 'function') {
        callback = this.noopFunc;
    }

    let loadHostsFilesEnd = function () {
        ηm.ubiquitousBlacklist.freeze();
        vAPI.storage.set({
	    'liveHostsFiles': ηm.liveHostsFiles
	});
        vAPI.messaging.broadcast({
	    what: 'loadHostsFilesCompleted'
	});
        ηm.getBytesInUse();
        callback();
    };

    let mergeHostsFile = function (details) {
        ηm.mergeHostsFile(details);
        hostsFileLoadCount -= 1;
        if (hostsFileLoadCount === 0) {
            loadHostsFilesEnd();
        }
    };

    let loadHostsFilesStart = function (hostsFiles) {
        ηm.liveHostsFiles = hostsFiles;
        ηm.ubiquitousBlacklist.reset();

        let locations = Object.keys(hostsFiles);
        hostsFileLoadCount = locations.length;

        // Load all hosts file which are not disabled.
        let location;
        while ((location = locations.pop())) {
            if (hostsFiles[location].off) {
                hostsFileLoadCount -= 1;
                continue;
            }
            ηm.assets.get(location, mergeHostsFile);
        }

        // https://github.com/gorhill/uMatrix/issues/2
        if (hostsFileLoadCount === 0) {
            loadHostsFilesEnd();
            return;
        }
    };

    this.getAvailableHostsFiles(loadHostsFilesStart);
};

ηMatrix.mergeHostsFile = function (details) {
    let usedCount = this.ubiquitousBlacklist.count;
    let duplicateCount = this.ubiquitousBlacklist.duplicateCount;

    this.mergeHostsFileContent(details.content);

    usedCount = this.ubiquitousBlacklist.count - usedCount;
    duplicateCount = this.ubiquitousBlacklist.duplicateCount - duplicateCount;

    var hostsFilesMeta = this.liveHostsFiles[details.assetKey];
    hostsFilesMeta.entryCount = usedCount + duplicateCount;
    hostsFilesMeta.entryUsedCount = usedCount;
};

ηMatrix.mergeHostsFileContent = function (rawText) {
    let rawEnd = rawText.length;
    let ubiquitousBlacklist = this.ubiquitousBlacklist;
    let reLocalhost = /(^|\s)(localhost\.localdomain|localhost|local|broadcasthost|0\.0\.0\.0|127\.0\.0\.1|::1|fe80::1%lo0)(?=\s|$)/g;
    let reAsciiSegment = /^[\x21-\x7e]+$/;
    let matches;
    let lineBeg = 0, lineEnd;
    let line;

    while (lineBeg < rawEnd) {
        lineEnd = rawText.indexOf('\n', lineBeg);
        if (lineEnd < 0) {
            lineEnd = rawText.indexOf('\r', lineBeg);
            if (lineEnd < 0) {
                lineEnd = rawEnd;
            }
        }

        // rhill 2014-04-18: The trim is important here, as without it there
        // could be a lingering `\r` which would cause problems in the
        // following parsing code.
        line = rawText.slice(lineBeg, lineEnd).trim();
        lineBeg = lineEnd + 1;

        // https://github.com/gorhill/httpswitchboard/issues/15
        // Ensure localhost et al. don't end up in the ubiquitous blacklist.
        line = line
            .replace(/#.*$/, '')
            .toLowerCase()
            .replace(reLocalhost, '')
            .trim();

        // The filter is whatever sequence of printable ascii character without
        // whitespaces
        matches = reAsciiSegment.exec(line);
        if (!matches || matches.length === 0) {
            continue;
        }

        // Bypass anomalies
        // For example, when a filter contains whitespace characters, or
        // whatever else outside the range of printable ascii characters.
        if (matches[0] !== line) {
            continue;
        }

        line = matches[0];
        if (line === '') {
            continue;
        }

        ubiquitousBlacklist.add(line);
    }
};

// `switches` contains the filter lists for which the switch must be revisited.
ηMatrix.selectHostsFiles = function (details, callback) {
    let ηm = this;
    let externalHostsFiles = this.userSettings.externalHostsFiles;

    // Hosts file to select
    if (Array.isArray(details.toSelect)) {
        for (let assetKey in this.liveHostsFiles) {
            if (this.liveHostsFiles.hasOwnProperty(assetKey) === false) {
                continue;
            }
            if (details.toSelect.indexOf(assetKey) !== -1) {
                this.liveHostsFiles[assetKey].off = false;
            } else if ( details.merge !== true ) {
                this.liveHostsFiles[assetKey].off = true;
            }
        }
    }

    // Imported hosts files to remove
    if (Array.isArray(details.toRemove)) {
        let removeURLFromHaystack = function (haystack, needle) {
            return haystack
		.replace(new RegExp('(^|\\n)'
				    + needle.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
				    + '(\\n|$)', 'g'),
			 '\n').trim();
        };

        for (let i=0, n=details.toRemove.length; i<n; ++i) {
            let assetKey = details.toRemove[i];
            delete this.liveHostsFiles[assetKey];
            externalHostsFiles = removeURLFromHaystack(externalHostsFiles, assetKey);
            this.assets.remove(assetKey);
        }
    }

    // Hosts file to import
    if (typeof details.toImport === 'string') {
        // https://github.com/gorhill/uBlock/issues/1181
        //   Try mapping the URL of an imported filter list to the assetKey of an
        //   existing stock list.
        let assetKeyFromURL = function (url) {
            let needle = url.replace(/^https?:/, '');
            let assets = ηm.liveHostsFiles;

            for (let assetKey in assets) {
                let asset = assets[assetKey];
                if (asset.content !== 'filters') {
		    continue;
		}
                if (typeof asset.contentURL === 'string') {
                    if (asset.contentURL.endsWith(needle)) {
			return assetKey;
		    }
                    continue;
                }
                if (Array.isArray(asset.contentURL) === false) {
		    continue;
		}
                for (let i=0, n=asset.contentURL.length; i<n; ++i) {
                    if (asset.contentURL[i].endsWith(needle)) {
                        return assetKey;
                    }
                }
            }

            return url;
        };

        let importedSet = new Set(this.listKeysFromCustomHostsFiles(externalHostsFiles));
        let toImportSet = new Set(this.listKeysFromCustomHostsFiles(details.toImport));
        let iter = toImportSet.values();

        for (;;) {
            let entry = iter.next();
            if (entry.done) {
		break;
	    }

            if (importedSet.has(entry.value)) {
		continue;
	    }

            let assetKey = assetKeyFromURL(entry.value);
            if (assetKey === entry.value) {
                importedSet.add(entry.value);
            }

            this.liveHostsFiles[assetKey] = {
                content: 'filters',
                contentURL: [ assetKey ],
                title: assetKey
            };
        }

        externalHostsFiles = Tools.setToArray(importedSet).sort().join('\n');
    }

    if (externalHostsFiles !== this.userSettings.externalHostsFiles) {
        this.userSettings.externalHostsFiles = externalHostsFiles;
        vAPI.storage.set({
	    externalHostsFiles: externalHostsFiles
	});
    }

    vAPI.storage.set({
	'liveHostsFiles': this.liveHostsFiles
    });

    if (typeof callback === 'function') {
        callback();
    }
};

// `switches` contains the preset blacklists for which the switch must be
// revisited.
ηMatrix.reloadHostsFiles = function () {
    this.loadHostsFiles();
};

ηMatrix.loadPublicSuffixList = function (callback) {
    if (typeof callback !== 'function') {
        callback = this.noopFunc;
    }

    let applyPublicSuffixList = function (details) {
        if (!details.error) {
            publicSuffixList.parse(details.content, Punycode.toASCII);
        }

        callback();
    };

    this.assets.get(this.pslAssetKey, applyPublicSuffixList);
};

ηMatrix.scheduleAssetUpdater = (function () {
    let timer = undefined;
    let next = 0;

    return function (updateDelay) {
        if (timer) {
            clearTimeout(timer);
            timer = undefined;
        }

        if (updateDelay === 0) {
            next = 0;
            return;
        }

        let now = Date.now();
        // Use the new schedule if and only if it is earlier than the
        // previous one.
        if (next !== 0) {
            updateDelay = Math.min(updateDelay, Math.max(next - now, 0));
        }

        next = now + updateDelay;
        timer = vAPI.setTimeout(function () {
            timer = undefined;
            next = 0;
            ηMatrix.assets.updateStart({
		delay: 120000
	    });
        }, updateDelay);
    };
})();

ηMatrix.assetObserver = function (topic, details) {
    // Do not update filter list if not in use.
    if (topic === 'before-asset-updated') {
        if (this.liveHostsFiles.hasOwnProperty(details.assetKey) === false ||
	    this.liveHostsFiles[details.assetKey].off === true) {
            return false;
        }
        return true;
    }

    if (topic === 'after-asset-updated') {
        vAPI.messaging.broadcast({
            what: 'assetUpdated',
            key: details.assetKey,
            cached: true
        });
        return;
    }

    // Update failed.
    if (topic === 'asset-update-failed') {
        vAPI.messaging.broadcast({
            what: 'assetUpdated',
            key: details.assetKey,
            failed: true
        });
        return;
    }

    // Reload all filter lists if needed.
    if (topic === 'after-assets-updated') {
        if (details.assetKeys.length !== 0) {
            this.loadHostsFiles();
        }

	ηm.scheduleAssetUpdater(ηm.userSettings.autoUpdate ?
				7 * 60 * 100000 :
				0);

        vAPI.messaging.broadcast({
            what: 'assetsUpdated',
            assetKeys: details.assetKeys
        });
        return;
    }

    // New asset source became available, if it's a filter list, should we
    // auto-select it?
    if (topic === 'builtin-asset-source-added') {
        if (details.entry.content === 'filters') {
            if (details.entry.off !== true) {
                this.saveSelectedFilterLists([ details.assetKey ], true);
            }
        }
        return;
    }
};