diff options
Diffstat (limited to 'search')
| -rw-r--r-- | search/lunr.js | 3475 | ||||
| -rw-r--r-- | search/main.js | 109 | ||||
| -rw-r--r-- | search/search_index.json | 1 | ||||
| -rw-r--r-- | search/worker.js | 133 |
4 files changed, 3718 insertions, 0 deletions
diff --git a/search/lunr.js b/search/lunr.js new file mode 100644 index 0000000..30b493e --- /dev/null +++ b/search/lunr.js @@ -0,0 +1,3475 @@ +/** + * lunr - http://lunrjs.com - A bit like Solr, but much smaller and not as bright - 2.3.9 + * Copyright (C) 2022 Oliver Nightingale + * @license MIT + */ + +;(function(){ + +/** + * A convenience function for configuring and constructing + * a new lunr Index. + * + * A lunr.Builder instance is created and the pipeline setup + * with a trimmer, stop word filter and stemmer. + * + * This builder object is yielded to the configuration function + * that is passed as a parameter, allowing the list of fields + * and other builder parameters to be customised. + * + * All documents _must_ be added within the passed config function. + * + * @example + * var idx = lunr(function () { + * this.field('title') + * this.field('body') + * this.ref('id') + * + * documents.forEach(function (doc) { + * this.add(doc) + * }, this) + * }) + * + * @see {@link lunr.Builder} + * @see {@link lunr.Pipeline} + * @see {@link lunr.trimmer} + * @see {@link lunr.stopWordFilter} + * @see {@link lunr.stemmer} + * @namespace {function} lunr + */ +var lunr = function (config) { + var builder = new lunr.Builder + + builder.pipeline.add( + lunr.trimmer, + lunr.stopWordFilter, + lunr.stemmer + ) + + builder.searchPipeline.add( + lunr.stemmer + ) + + config.call(builder, builder) + return builder.build() +} + +lunr.version = "2.3.9" +/*! + * lunr.utils + * Copyright (C) 2022 Oliver Nightingale + */ + +/** + * A namespace containing utils for the rest of the lunr library + * @namespace lunr.utils + */ +lunr.utils = {} + +/** + * Print a warning message to the console. + * + * @param {String} message The message to be printed. + * @memberOf lunr.utils + * @function + */ +lunr.utils.warn = (function (global) { + /* eslint-disable no-console */ + return function (message) { + if (global.console && console.warn) { + console.warn(message) + } + } + /* eslint-enable no-console */ +})(this) + +/** + * Convert an object to a string. + * + * In the case of `null` and `undefined` the function returns + * the empty string, in all other cases the result of calling + * `toString` on the passed object is returned. + * + * @param {Any} obj The object to convert to a string. + * @return {String} string representation of the passed object. + * @memberOf lunr.utils + */ +lunr.utils.asString = function (obj) { + if (obj === void 0 || obj === null) { + return "" + } else { + return obj.toString() + } +} + +/** + * Clones an object. + * + * Will create a copy of an existing object such that any mutations + * on the copy cannot affect the original. + * + * Only shallow objects are supported, passing a nested object to this + * function will cause a TypeError. + * + * Objects with primitives, and arrays of primitives are supported. + * + * @param {Object} obj The object to clone. + * @return {Object} a clone of the passed object. + * @throws {TypeError} when a nested object is passed. + * @memberOf Utils + */ +lunr.utils.clone = function (obj) { + if (obj === null || obj === undefined) { + return obj + } + + var clone = Object.create(null), + keys = Object.keys(obj) + + for (var i = 0; i < keys.length; i++) { + var key = keys[i], + val = obj[key] + + if (Array.isArray(val)) { + clone[key] = val.slice() + continue + } + + if (typeof val === 'string' || + typeof val === 'number' || + typeof val === 'boolean') { + clone[key] = val + continue + } + + throw new TypeError("clone is not deep and does not support nested objects") + } + + return clone +} +lunr.FieldRef = function (docRef, fieldName, stringValue) { + this.docRef = docRef + this.fieldName = fieldName + this._stringValue = stringValue +} + +lunr.FieldRef.joiner = "/" + +lunr.FieldRef.fromString = function (s) { + var n = s.indexOf(lunr.FieldRef.joiner) + + if (n === -1) { + throw "malformed field ref string" + } + + var fieldRef = s.slice(0, n), + docRef = s.slice(n + 1) + + return new lunr.FieldRef (docRef, fieldRef, s) +} + +lunr.FieldRef.prototype.toString = function () { + if (this._stringValue == undefined) { + this._stringValue = this.fieldName + lunr.FieldRef.joiner + this.docRef + } + + return this._stringValue +} +/*! + * lunr.Set + * Copyright (C) 2022 Oliver Nightingale + */ + +/** + * A lunr set. + * + * @constructor + */ +lunr.Set = function (elements) { + this.elements = Object.create(null) + + if (elements) { + this.length = elements.length + + for (var i = 0; i < this.length; i++) { + this.elements[elements[i]] = true + } + } else { + this.length = 0 + } +} + +/** + * A complete set that contains all elements. + * + * @static + * @readonly + * @type {lunr.Set} + */ +lunr.Set.complete = { + intersect: function (other) { + return other + }, + + union: function () { + return this + }, + + contains: function () { + return true + } +} + +/** + * An empty set that contains no elements. + * + * @static + * @readonly + * @type {lunr.Set} + */ +lunr.Set.empty = { + intersect: function () { + return this + }, + + union: function (other) { + return other + }, + + contains: function () { + return false + } +} + +/** + * Returns true if this set contains the specified object. + * + * @param {object} object - Object whose presence in this set is to be tested. + * @returns {boolean} - True if this set contains the specified object. + */ +lunr.Set.prototype.contains = function (object) { + return !!this.elements[object] +} + +/** + * Returns a new set containing only the elements that are present in both + * this set and the specified set. + * + * @param {lunr.Set} other - set to intersect with this set. + * @returns {lunr.Set} a new set that is the intersection of this and the specified set. + */ + +lunr.Set.prototype.intersect = function (other) { + var a, b, elements, intersection = [] + + if (other === lunr.Set.complete) { + return this + } + + if (other === lunr.Set.empty) { + return other + } + + if (this.length < other.length) { + a = this + b = other + } else { + a = other + b = this + } + + elements = Object.keys(a.elements) + + for (var i = 0; i < elements.length; i++) { + var element = elements[i] + if (element in b.elements) { + intersection.push(element) + } + } + + return new lunr.Set (intersection) +} + +/** + * Returns a new set combining the elements of this and the specified set. + * + * @param {lunr.Set} other - set to union with this set. + * @return {lunr.Set} a new set that is the union of this and the specified set. + */ + +lunr.Set.prototype.union = function (other) { + if (other === lunr.Set.complete) { + return lunr.Set.complete + } + + if (other === lunr.Set.empty) { + return this + } + + return new lunr.Set(Object.keys(this.elements).concat(Object.keys(other.elements))) +} +/** + * A function to calculate the inverse document frequency for + * a posting. This is shared between the builder and the index + * + * @private + * @param {object} posting - The posting for a given term + * @param {number} documentCount - The total number of documents. + */ +lunr.idf = function (posting, documentCount) { + var documentsWithTerm = 0 + + for (var fieldName in posting) { + if (fieldName == '_index') continue // Ignore the term index, its not a field + documentsWithTerm += Object.keys(posting[fieldName]).length + } + + var x = (documentCount - documentsWithTerm + 0.5) / (documentsWithTerm + 0.5) + + return Math.log(1 + Math.abs(x)) +} + +/** + * A token wraps a string representation of a token + * as it is passed through the text processing pipeline. + * + * @constructor + * @param {string} [str=''] - The string token being wrapped. + * @param {object} [metadata={}] - Metadata associated with this token. + */ +lunr.Token = function (str, metadata) { + this.str = str || "" + this.metadata = metadata || {} +} + +/** + * Returns the token string that is being wrapped by this object. + * + * @returns {string} + */ +lunr.Token.prototype.toString = function () { + return this.str +} + +/** + * A token update function is used when updating or optionally + * when cloning a token. + * + * @callback lunr.Token~updateFunction + * @param {string} str - The string representation of the token. + * @param {Object} metadata - All metadata associated with this token. + */ + +/** + * Applies the given function to the wrapped string token. + * + * @example + * token.update(function (str, metadata) { + * return str.toUpperCase() + * }) + * + * @param {lunr.Token~updateFunction} fn - A function to apply to the token string. + * @returns {lunr.Token} + */ +lunr.Token.prototype.update = function (fn) { + this.str = fn(this.str, this.metadata) + return this +} + +/** + * Creates a clone of this token. Optionally a function can be + * applied to the cloned token. + * + * @param {lunr.Token~updateFunction} [fn] - An optional function to apply to the cloned token. + * @returns {lunr.Token} + */ +lunr.Token.prototype.clone = function (fn) { + fn = fn || function (s) { return s } + return new lunr.Token (fn(this.str, this.metadata), this.metadata) +} +/*! + * lunr.tokenizer + * Copyright (C) 2022 Oliver Nightingale + */ + +/** + * A function for splitting a string into tokens ready to be inserted into + * the search index. Uses `lunr.tokenizer.separator` to split strings, change + * the value of this property to change how strings are split into tokens. + * + * This tokenizer will convert its parameter to a string by calling `toString` and + * then will split this string on the character in `lunr.tokenizer.separator`. + * Arrays will have their elements converted to strings and wrapped in a lunr.Token. + * + * Optional metadata can be passed to the tokenizer, this metadata will be cloned and + * added as metadata to every token that is created from the object to be tokenized. + * + * @static + * @param {?(string|object|object[])} obj - The object to convert into tokens + * @param {?object} metadata - Optional metadata to associate with every token + * @returns {lunr.Token[]} + * @see {@link lunr.Pipeline} + */ +lunr.tokenizer = function (obj, metadata) { + if (obj == null || obj == undefined) { + return [] + } + + if (Array.isArray(obj)) { + return obj.map(function (t) { + return new lunr.Token( + lunr.utils.asString(t).toLowerCase(), + lunr.utils.clone(metadata) + ) + }) + } + + var str = obj.toString().toLowerCase(), + len = str.length, + tokens = [] + + for (var sliceEnd = 0, sliceStart = 0; sliceEnd <= len; sliceEnd++) { + var char = str.charAt(sliceEnd), + sliceLength = sliceEnd - sliceStart + + if ((char.match(lunr.tokenizer.separator) || sliceEnd == len)) { + + if (sliceLength > 0) { + var tokenMetadata = lunr.utils.clone(metadata) || {} + tokenMetadata["position"] = [sliceStart, sliceLength] + tokenMetadata["index"] = tokens.length + + tokens.push( + new lunr.Token ( + str.slice(sliceStart, sliceEnd), + tokenMetadata + ) + ) + } + + sliceStart = sliceEnd + 1 + } + + } + + return tokens +} + +/** + * The separator used to split a string into tokens. Override this property to change the behaviour of + * `lunr.tokenizer` behaviour when tokenizing strings. By default this splits on whitespace and hyphens. + * + * @static + * @see lunr.tokenizer + */ +lunr.tokenizer.separator = /[\s\-]+/ +/*! + * lunr.Pipeline + * Copyright (C) 2022 Oliver Nightingale + */ + +/** + * lunr.Pipelines maintain an ordered list of functions to be applied to all + * tokens in documents entering the search index and queries being ran against + * the index. + * + * An instance of lunr.Index created with the lunr shortcut will contain a + * pipeline with a stop word filter and an English language stemmer. Extra + * functions can be added before or after either of these functions or these + * default functions can be removed. + * + * When run the pipeline will call each function in turn, passing a token, the + * index of that token in the original list of all tokens and finally a list of + * all the original tokens. + * + * The output of functions in the pipeline will be passed to the next function + * in the pipeline. To exclude a token from entering the index the function + * should return undefined, the rest of the pipeline will not be called with + * this token. + * + * For serialisation of pipelines to work, all functions used in an instance of + * a pipeline should be registered with lunr.Pipeline. Registered functions can + * then be loaded. If trying to load a serialised pipeline that uses functions + * that are not registered an error will be thrown. + * + * If not planning on serialising the pipeline then registering pipeline functions + * is not necessary. + * + * @constructor + */ +lunr.Pipeline = function () { + this._stack = [] +} + +lunr.Pipeline.registeredFunctions = Object.create(null) + +/** + * A pipeline function maps lunr.Token to lunr.Token. A lunr.Token contains the token + * string as well as all known metadata. A pipeline function can mutate the token string + * or mutate (or add) metadata for a given token. + * + * A pipeline function can indicate that the passed token should be discarded by returning + * null, undefined or an empty string. This token will not be passed to any downstream pipeline + * functions and will not be added to the index. + * + * Multiple tokens can be returned by returning an array of tokens. Each token will be passed + * to any downstream pipeline functions and all will returned tokens will be added to the index. + * + * Any number of pipeline functions may be chained together using a lunr.Pipeline. + * + * @interface lunr.PipelineFunction + * @param {lunr.Token} token - A token from the document being processed. + * @param {number} i - The index of this token in the complete list of tokens for this document/field. + * @param {lunr.Token[]} tokens - All tokens for this document/field. + * @returns {(?lunr.Token|lunr.Token[])} + */ + +/** + * Register a function with the pipeline. + * + * Functions that are used in the pipeline should be registered if the pipeline + * needs to be serialised, or a serialised pipeline needs to be loaded. + * + * Registering a function does not add it to a pipeline, functions must still be + * added to instances of the pipeline for them to be used when running a pipeline. + * + * @param {lunr.PipelineFunction} fn - The function to check for. + * @param {String} label - The label to register this function with + */ +lunr.Pipeline.registerFunction = function (fn, label) { + if (label in this.registeredFunctions) { + lunr.utils.warn('Overwriting existing registered function: ' + label) + } + + fn.label = label + lunr.Pipeline.registeredFunctions[fn.label] = fn +} + +/** + * Warns if the function is not registered as a Pipeline function. + * + * @param {lunr.PipelineFunction} fn - The function to check for. + * @private + */ +lunr.Pipeline.warnIfFunctionNotRegistered = function (fn) { + var isRegistered = fn.label && (fn.label in this.registeredFunctions) + + if (!isRegistered) { + lunr.utils.warn('Function is not registered with pipeline. This may cause problems when serialising the index.\n', fn) + } +} + +/** + * Loads a previously serialised pipeline. + * + * All functions to be loaded must already be registered with lunr.Pipeline. + * If any function from the serialised data has not been registered then an + * error will be thrown. + * + * @param {Object} serialised - The serialised pipeline to load. + * @returns {lunr.Pipeline} + */ +lunr.Pipeline.load = function (serialised) { + var pipeline = new lunr.Pipeline + + serialised.forEach(function (fnName) { + var fn = lunr.Pipeline.registeredFunctions[fnName] + + if (fn) { + pipeline.add(fn) + } else { + throw new Error('Cannot load unregistered function: ' + fnName) + } + }) + + return pipeline +} + +/** + * Adds new functions to the end of the pipeline. + * + * Logs a warning if the function has not been registered. + * + * @param {lunr.PipelineFunction[]} functions - Any number of functions to add to the pipeline. + */ +lunr.Pipeline.prototype.add = function () { + var fns = Array.prototype.slice.call(arguments) + + fns.forEach(function (fn) { + lunr.Pipeline.warnIfFunctionNotRegistered(fn) + this._stack.push(fn) + }, this) +} + +/** + * Adds a single function after a function that already exists in the + * pipeline. + * + * Logs a warning if the function has not been registered. + * + * @param {lunr.PipelineFunction} existingFn - A function that already exists in the pipeline. + * @param {lunr.PipelineFunction} newFn - The new function to add to the pipeline. + */ +lunr.Pipeline.prototype.after = function (existingFn, newFn) { + lunr.Pipeline.warnIfFunctionNotRegistered(newFn) + + var pos = this._stack.indexOf(existingFn) + if (pos == -1) { + throw new Error('Cannot find existingFn') + } + + pos = pos + 1 + this._stack.splice(pos, 0, newFn) +} + +/** + * Adds a single function before a function that already exists in the + * pipeline. + * + * Logs a warning if the function has not been registered. + * + * @param {lunr.PipelineFunction} existingFn - A function that already exists in the pipeline. + * @param {lunr.PipelineFunction} newFn - The new function to add to the pipeline. + */ +lunr.Pipeline.prototype.before = function (existingFn, newFn) { + lunr.Pipeline.warnIfFunctionNotRegistered(newFn) + + var pos = this._stack.indexOf(existingFn) + if (pos == -1) { + throw new Error('Cannot find existingFn') + } + + this._stack.splice(pos, 0, newFn) +} + +/** + * Removes a function from the pipeline. + * + * @param {lunr.PipelineFunction} fn The function to remove from the pipeline. + */ +lunr.Pipeline.prototype.remove = function (fn) { + var pos = this._stack.indexOf(fn) + if (pos == -1) { + return + } + + this._stack.splice(pos, 1) +} + +/** + * Runs the current list of functions that make up the pipeline against the + * passed tokens. + * + * @param {Array} tokens The tokens to run through the pipeline. + * @returns {Array} + */ +lunr.Pipeline.prototype.run = function (tokens) { + var stackLength = this._stack.length + + for (var i = 0; i < stackLength; i++) { + var fn = this._stack[i] + var memo = [] + + for (var j = 0; j < tokens.length; j++) { + var result = fn(tokens[j], j, tokens) + + if (result === null || result === void 0 || result === '') continue + + if (Array.isArray(result)) { + for (var k = 0; k < result.length; k++) { + memo.push(result[k]) + } + } else { + memo.push(result) + } + } + + tokens = memo + } + + return tokens +} + +/** + * Convenience method for passing a string through a pipeline and getting + * strings out. This method takes care of wrapping the passed string in a + * token and mapping the resulting tokens back to strings. + * + * @param {string} str - The string to pass through the pipeline. + * @param {?object} metadata - Optional metadata to associate with the token + * passed to the pipeline. + * @returns {string[]} + */ +lunr.Pipeline.prototype.runString = function (str, metadata) { + var token = new lunr.Token (str, metadata) + + return this.run([token]).map(function (t) { + return t.toString() + }) +} + +/** + * Resets the pipeline by removing any existing processors. + * + */ +lunr.Pipeline.prototype.reset = function () { + this._stack = [] +} + +/** + * Returns a representation of the pipeline ready for serialisation. + * + * Logs a warning if the function has not been registered. + * + * @returns {Array} + */ +lunr.Pipeline.prototype.toJSON = function () { + return this._stack.map(function (fn) { + lunr.Pipeline.warnIfFunctionNotRegistered(fn) + + return fn.label + }) +} +/*! + * lunr.Vector + * Copyright (C) 2022 Oliver Nightingale + */ + +/** + * A vector is used to construct the vector space of documents and queries. These + * vectors support operations to determine the similarity between two documents or + * a document and a query. + * + * Normally no parameters are required for initializing a vector, but in the case of + * loading a previously dumped vector the raw elements can be provided to the constructor. + * + * For performance reasons vectors are implemented with a flat array, where an elements + * index is immediately followed by its value. E.g. [index, value, index, value]. This + * allows the underlying array to be as sparse as possible and still offer decent + * performance when being used for vector calculations. + * + * @constructor + * @param {Number[]} [elements] - The flat list of element index and element value pairs. + */ +lunr.Vector = function (elements) { + this._magnitude = 0 + this.elements = elements || [] +} + + +/** + * Calculates the position within the vector to insert a given index. + * + * This is used internally by insert and upsert. If there are duplicate indexes then + * the position is returned as if the value for that index were to be updated, but it + * is the callers responsibility to check whether there is a duplicate at that index + * + * @param {Number} insertIdx - The index at which the element should be inserted. + * @returns {Number} + */ +lunr.Vector.prototype.positionForIndex = function (index) { + // For an empty vector the tuple can be inserted at the beginning + if (this.elements.length == 0) { + return 0 + } + + var start = 0, + end = this.elements.length / 2, + sliceLength = end - start, + pivotPoint = Math.floor(sliceLength / 2), + pivotIndex = this.elements[pivotPoint * 2] + + while (sliceLength > 1) { + if (pivotIndex < index) { + start = pivotPoint + } + + if (pivotIndex > index) { + end = pivotPoint + } + + if (pivotIndex == index) { + break + } + + sliceLength = end - start + pivotPoint = start + Math.floor(sliceLength / 2) + pivotIndex = this.elements[pivotPoint * 2] + } + + if (pivotIndex == index) { + return pivotPoint * 2 + } + + if (pivotIndex > index) { + return pivotPoint * 2 + } + + if (pivotIndex < index) { + return (pivotPoint + 1) * 2 + } +} + +/** + * Inserts an element at an index within the vector. + * + * Does not allow duplicates, will throw an error if there is already an entry + * for this index. + * + * @param {Number} insertIdx - The index at which the element should be inserted. + * @param {Number} val - The value to be inserted into the vector. + */ +lunr.Vector.prototype.insert = function (insertIdx, val) { + this.upsert(insertIdx, val, function () { + throw "duplicate index" + }) +} + +/** + * Inserts or updates an existing index within the vector. + * + * @param {Number} insertIdx - The index at which the element should be inserted. + * @param {Number} val - The value to be inserted into the vector. + * @param {function} fn - A function that is called for updates, the existing value and the + * requested value are passed as arguments + */ +lunr.Vector.prototype.upsert = function (insertIdx, val, fn) { + this._magnitude = 0 + var position = this.positionForIndex(insertIdx) + + if (this.elements[position] == insertIdx) { + this.elements[position + 1] = fn(this.elements[position + 1], val) + } else { + this.elements.splice(position, 0, insertIdx, val) + } +} + +/** + * Calculates the magnitude of this vector. + * + * @returns {Number} + */ +lunr.Vector.prototype.magnitude = function () { + if (this._magnitude) return this._magnitude + + var sumOfSquares = 0, + elementsLength = this.elements.length + + for (var i = 1; i < elementsLength; i += 2) { + var val = this.elements[i] + sumOfSquares += val * val + } + + return this._magnitude = Math.sqrt(sumOfSquares) +} + +/** + * Calculates the dot product of this vector and another vector. + * + * @param {lunr.Vector} otherVector - The vector to compute the dot product with. + * @returns {Number} + */ +lunr.Vector.prototype.dot = function (otherVector) { + var dotProduct = 0, + a = this.elements, b = otherVector.elements, + aLen = a.length, bLen = b.length, + aVal = 0, bVal = 0, + i = 0, j = 0 + + while (i < aLen && j < bLen) { + aVal = a[i], bVal = b[j] + if (aVal < bVal) { + i += 2 + } else if (aVal > bVal) { + j += 2 + } else if (aVal == bVal) { + dotProduct += a[i + 1] * b[j + 1] + i += 2 + j += 2 + } + } + + return dotProduct +} + +/** + * Calculates the similarity between this vector and another vector. + * + * @param {lunr.Vector} otherVector - The other vector to calculate the + * similarity with. + * @returns {Number} + */ +lunr.Vector.prototype.similarity = function (otherVector) { + return this.dot(otherVector) / this.magnitude() || 0 +} + +/** + * Converts the vector to an array of the elements within the vector. + * + * @returns {Number[]} + */ +lunr.Vector.prototype.toArray = function () { + var output = new Array (this.elements.length / 2) + + for (var i = 1, j = 0; i < this.elements.length; i += 2, j++) { + output[j] = this.elements[i] + } + + return output +} + +/** + * A JSON serializable representation of the vector. + * + * @returns {Number[]} + */ +lunr.Vector.prototype.toJSON = function () { + return this.elements +} +/* eslint-disable */ +/*! + * lunr.stemmer + * Copyright (C) 2022 Oliver Nightingale + * Includes code from - http://tartarus.org/~martin/PorterStemmer/js.txt + */ + +/** + * lunr.stemmer is an english language stemmer, this is a JavaScript + * implementation of the PorterStemmer taken from http://tartarus.org/~martin + * + * @static + * @implements {lunr.PipelineFunction} + * @param {lunr.Token} token - The string to stem + * @returns {lunr.Token} + * @see {@link lunr.Pipeline} + * @function + */ +lunr.stemmer = (function(){ + var step2list = { + "ational" : "ate", + "tional" : "tion", + "enci" : "ence", + "anci" : "ance", + "izer" : "ize", + "bli" : "ble", + "alli" : "al", + "entli" : "ent", + "eli" : "e", + "ousli" : "ous", + "ization" : "ize", + "ation" : "ate", + "ator" : "ate", + "alism" : "al", + "iveness" : "ive", + "fulness" : "ful", + "ousness" : "ous", + "aliti" : "al", + "iviti" : "ive", + "biliti" : "ble", + "logi" : "log" + }, + + step3list = { + "icate" : "ic", + "ative" : "", + "alize" : "al", + "iciti" : "ic", + "ical" : "ic", + "ful" : "", + "ness" : "" + }, + + c = "[^aeiou]", // consonant + v = "[aeiouy]", // vowel + C = c + "[^aeiouy]*", // consonant sequence + V = v + "[aeiou]*", // vowel sequence + + mgr0 = "^(" + C + ")?" + V + C, // [C]VC... is m>0 + meq1 = "^(" + C + ")?" + V + C + "(" + V + ")?$", // [C]VC[V] is m=1 + mgr1 = "^(" + C + ")?" + V + C + V + C, // [C]VCVC... is m>1 + s_v = "^(" + C + ")?" + v; // vowel in stem + + var re_mgr0 = new RegExp(mgr0); + var re_mgr1 = new RegExp(mgr1); + var re_meq1 = new RegExp(meq1); + var re_s_v = new RegExp(s_v); + + var re_1a = /^(.+?)(ss|i)es$/; + var re2_1a = /^(.+?)([^s])s$/; + var re_1b = /^(.+?)eed$/; + var re2_1b = /^(.+?)(ed|ing)$/; + var re_1b_2 = /.$/; + var re2_1b_2 = /(at|bl|iz)$/; + var re3_1b_2 = new RegExp("([^aeiouylsz])\\1$"); + var re4_1b_2 = new RegExp("^" + C + v + "[^aeiouwxy]$"); + + var re_1c = /^(.+?[^aeiou])y$/; + var re_2 = /^(.+?)(ational|tional|enci|anci|izer|bli|alli|entli|eli|ousli|ization|ation|ator|alism|iveness|fulness|ousness|aliti|iviti|biliti|logi)$/; + + var re_3 = /^(.+?)(icate|ative|alize|iciti|ical|ful|ness)$/; + + var re_4 = /^(.+?)(al|ance|ence|er|ic|able|ible|ant|ement|ment|ent|ou|ism|ate|iti|ous|ive|ize)$/; + var re2_4 = /^(.+?)(s|t)(ion)$/; + + var re_5 = /^(.+?)e$/; + var re_5_1 = /ll$/; + var re3_5 = new RegExp("^" + C + v + "[^aeiouwxy]$"); + + var porterStemmer = function porterStemmer(w) { + var stem, + suffix, + firstch, + re, + re2, + re3, + re4; + + if (w.length < 3) { return w; } + + firstch = w.substr(0,1); + if (firstch == "y") { + w = firstch.toUpperCase() + w.substr(1); + } + + // Step 1a + re = re_1a + re2 = re2_1a; + + if (re.test(w)) { w = w.replace(re,"$1$2"); } + else if (re2.test(w)) { w = w.replace(re2,"$1$2"); } + + // Step 1b + re = re_1b; + re2 = re2_1b; + if (re.test(w)) { + var fp = re.exec(w); + re = re_mgr0; + if (re.test(fp[1])) { + re = re_1b_2; + w = w.replace(re,""); + } + } else if (re2.test(w)) { + var fp = re2.exec(w); + stem = fp[1]; + re2 = re_s_v; + if (re2.test(stem)) { + w = stem; + re2 = re2_1b_2; + re3 = re3_1b_2; + re4 = re4_1b_2; + if (re2.test(w)) { w = w + "e"; } + else if (re3.test(w)) { re = re_1b_2; w = w.replace(re,""); } + else if (re4.test(w)) { w = w + "e"; } + } + } + + // Step 1c - replace suffix y or Y by i if preceded by a non-vowel which is not the first letter of the word (so cry -> cri, by -> by, say -> say) + re = re_1c; + if (re.test(w)) { + var fp = re.exec(w); + stem = fp[1]; + w = stem + "i"; + } + + // Step 2 + re = re_2; + if (re.test(w)) { + var fp = re.exec(w); + stem = fp[1]; + suffix = fp[2]; + re = re_mgr0; + if (re.test(stem)) { + w = stem + step2list[suffix]; + } + } + + // Step 3 + re = re_3; + if (re.test(w)) { + var fp = re.exec(w); + stem = fp[1]; + suffix = fp[2]; + re = re_mgr0; + if (re.test(stem)) { + w = stem + step3list[suffix]; + } + } + + // Step 4 + re = re_4; + re2 = re2_4; + if (re.test(w)) { + var fp = re.exec(w); + stem = fp[1]; + re = re_mgr1; + if (re.test(stem)) { + w = stem; + } + } else if (re2.test(w)) { + var fp = re2.exec(w); + stem = fp[1] + fp[2]; + re2 = re_mgr1; + if (re2.test(stem)) { + w = stem; + } + } + + // Step 5 + re = re_5; + if (re.test(w)) { + var fp = re.exec(w); + stem = fp[1]; + re = re_mgr1; + re2 = re_meq1; + re3 = re3_5; + if (re.test(stem) || (re2.test(stem) && !(re3.test(stem)))) { + w = stem; + } + } + + re = re_5_1; + re2 = re_mgr1; + if (re.test(w) && re2.test(w)) { + re = re_1b_2; + w = w.replace(re,""); + } + + // and turn initial Y back to y + + if (firstch == "y") { + w = firstch.toLowerCase() + w.substr(1); + } + + return w; + }; + + return function (token) { + return token.update(porterStemmer); + } +})(); + +lunr.Pipeline.registerFunction(lunr.stemmer, 'stemmer') +/*! + * lunr.stopWordFilter + * Copyright (C) 2022 Oliver Nightingale + */ + +/** + * lunr.generateStopWordFilter builds a stopWordFilter function from the provided + * list of stop words. + * + * The built in lunr.stopWordFilter is built using this generator and can be used + * to generate custom stopWordFilters for applications or non English languages. + * + * @function + * @param {Array} token The token to pass through the filter + * @returns {lunr.PipelineFunction} + * @see lunr.Pipeline + * @see lunr.stopWordFilter + */ +lunr.generateStopWordFilter = function (stopWords) { + var words = stopWords.reduce(function (memo, stopWord) { + memo[stopWord] = stopWord + return memo + }, {}) + + return function (token) { + if (token && words[token.toString()] !== token.toString()) return token + } +} + +/** + * lunr.stopWordFilter is an English language stop word list filter, any words + * contained in the list will not be passed through the filter. + * + * This is intended to be used in the Pipeline. If the token does not pass the + * filter then undefined will be returned. + * + * @function + * @implements {lunr.PipelineFunction} + * @params {lunr.Token} token - A token to check for being a stop word. + * @returns {lunr.Token} + * @see {@link lunr.Pipeline} + */ +lunr.stopWordFilter = lunr.generateStopWordFilter([ + 'a', + 'able', + 'about', + 'across', + 'after', + 'all', + 'almost', + 'also', + 'am', + 'among', + 'an', + 'and', + 'any', + 'are', + 'as', + 'at', + 'be', + 'because', + 'been', + 'but', + 'by', + 'can', + 'cannot', + 'could', + 'dear', + 'did', + 'do', + 'does', + 'either', + 'else', + 'ever', + 'every', + 'for', + 'from', + 'get', + 'got', + 'had', + 'has', + 'have', + 'he', + 'her', + 'hers', + 'him', + 'his', + 'how', + 'however', + 'i', + 'if', + 'in', + 'into', + 'is', + 'it', + 'its', + 'just', + 'least', + 'let', + 'like', + 'likely', + 'may', + 'me', + 'might', + 'most', + 'must', + 'my', + 'neither', + 'no', + 'nor', + 'not', + 'of', + 'off', + 'often', + 'on', + 'only', + 'or', + 'other', + 'our', + 'own', + 'rather', + 'said', + 'say', + 'says', + 'she', + 'should', + 'since', + 'so', + 'some', + 'than', + 'that', + 'the', + 'their', + 'them', + 'then', + 'there', + 'these', + 'they', + 'this', + 'tis', + 'to', + 'too', + 'twas', + 'us', + 'wants', + 'was', + 'we', + 'were', + 'what', + 'when', + 'where', + 'which', + 'while', + 'who', + 'whom', + 'why', + 'will', + 'with', + 'would', + 'yet', + 'you', + 'your' +]) + +lunr.Pipeline.registerFunction(lunr.stopWordFilter, 'stopWordFilter') +/*! + * lunr.trimmer + * Copyright (C) 2022 Oliver Nightingale + */ + +/** + * lunr.trimmer is a pipeline function for trimming non word + * characters from the beginning and end of tokens before they + * enter the index. + * + * This implementation may not work correctly for non latin + * characters and should either be removed or adapted for use + * with languages with non-latin characters. + * + * @static + * @implements {lunr.PipelineFunction} + * @param {lunr.Token} token The token to pass through the filter + * @returns {lunr.Token} + * @see lunr.Pipeline + */ +lunr.trimmer = function (token) { + return token.update(function (s) { + return s.replace(/^\W+/, '').replace(/\W+$/, '') + }) +} + +lunr.Pipeline.registerFunction(lunr.trimmer, 'trimmer') +/*! + * lunr.TokenSet + * Copyright (C) 2022 Oliver Nightingale + */ + +/** + * A token set is used to store the unique list of all tokens + * within an index. Token sets are also used to represent an + * incoming query to the index, this query token set and index + * token set are then intersected to find which tokens to look + * up in the inverted index. + * + * A token set can hold multiple tokens, as in the case of the + * index token set, or it can hold a single token as in the + * case of a simple query token set. + * + * Additionally token sets are used to perform wildcard matching. + * Leading, contained and trailing wildcards are supported, and + * from this edit distance matching can also be provided. + * + * Token sets are implemented as a minimal finite state automata, + * where both common prefixes and suffixes are shared between tokens. + * This helps to reduce the space used for storing the token set. + * + * @constructor + */ +lunr.TokenSet = function () { + this.final = false + this.edges = {} + this.id = lunr.TokenSet._nextId + lunr.TokenSet._nextId += 1 +} + +/** + * Keeps track of the next, auto increment, identifier to assign + * to a new tokenSet. + * + * TokenSets require a unique identifier to be correctly minimised. + * + * @private + */ +lunr.TokenSet._nextId = 1 + +/** + * Creates a TokenSet instance from the given sorted array of words. + * + * @param {String[]} arr - A sorted array of strings to create the set from. + * @returns {lunr.TokenSet} + * @throws Will throw an error if the input array is not sorted. + */ +lunr.TokenSet.fromArray = function (arr) { + var builder = new lunr.TokenSet.Builder + + for (var i = 0, len = arr.length; i < len; i++) { + builder.insert(arr[i]) + } + + builder.finish() + return builder.root +} + +/** + * Creates a token set from a query clause. + * + * @private + * @param {Object} clause - A single clause from lunr.Query. + * @param {string} clause.term - The query clause term. + * @param {number} [clause.editDistance] - The optional edit distance for the term. + * @returns {lunr.TokenSet} + */ +lunr.TokenSet.fromClause = function (clause) { + if ('editDistance' in clause) { + return lunr.TokenSet.fromFuzzyString(clause.term, clause.editDistance) + } else { + return lunr.TokenSet.fromString(clause.term) + } +} + +/** + * Creates a token set representing a single string with a specified + * edit distance. + * + * Insertions, deletions, substitutions and transpositions are each + * treated as an edit distance of 1. + * + * Increasing the allowed edit distance will have a dramatic impact + * on the performance of both creating and intersecting these TokenSets. + * It is advised to keep the edit distance less than 3. + * + * @param {string} str - The string to create the token set from. + * @param {number} editDistance - The allowed edit distance to match. + * @returns {lunr.Vector} + */ +lunr.TokenSet.fromFuzzyString = function (str, editDistance) { + var root = new lunr.TokenSet + + var stack = [{ + node: root, + editsRemaining: editDistance, + str: str + }] + + while (stack.length) { + var frame = stack.pop() + + // no edit + if (frame.str.length > 0) { + var char = frame.str.charAt(0), + noEditNode + + if (char in frame.node.edges) { + noEditNode = frame.node.edges[char] + } else { + noEditNode = new lunr.TokenSet + frame.node.edges[char] = noEditNode + } + + if (frame.str.length == 1) { + noEditNode.final = true + } + + stack.push({ + node: noEditNode, + editsRemaining: frame.editsRemaining, + str: frame.str.slice(1) + }) + } + + if (frame.editsRemaining == 0) { + continue + } + + // insertion + if ("*" in frame.node.edges) { + var insertionNode = frame.node.edges["*"] + } else { + var insertionNode = new lunr.TokenSet + frame.node.edges["*"] = insertionNode + } + + if (frame.str.length == 0) { + insertionNode.final = true + } + + stack.push({ + node: insertionNode, + editsRemaining: frame.editsRemaining - 1, + str: frame.str + }) + + // deletion + // can only do a deletion if we have enough edits remaining + // and if there are characters left to delete in the string + if (frame.str.length > 1) { + stack.push({ + node: frame.node, + editsRemaining: frame.editsRemaining - 1, + str: frame.str.slice(1) + }) + } + + // deletion + // just removing the last character from the str + if (frame.str.length == 1) { + frame.node.final = true + } + + // substitution + // can only do a substitution if we have enough edits remaining + // and if there are characters left to substitute + if (frame.str.length >= 1) { + if ("*" in frame.node.edges) { + var substitutionNode = frame.node.edges["*"] + } else { + var substitutionNode = new lunr.TokenSet + frame.node.edges["*"] = substitutionNode + } + + if (frame.str.length == 1) { + substitutionNode.final = true + } + + stack.push({ + node: substitutionNode, + editsRemaining: frame.editsRemaining - 1, + str: frame.str.slice(1) + }) + } + + // transposition + // can only do a transposition if there are edits remaining + // and there are enough characters to transpose + if (frame.str.length > 1) { + var charA = frame.str.charAt(0), + charB = frame.str.charAt(1), + transposeNode + + if (charB in frame.node.edges) { + transposeNode = frame.node.edges[charB] + } else { + transposeNode = new lunr.TokenSet + frame.node.edges[charB] = transposeNode + } + + if (frame.str.length == 1) { + transposeNode.final = true + } + + stack.push({ + node: transposeNode, + editsRemaining: frame.editsRemaining - 1, + str: charA + frame.str.slice(2) + }) + } + } + + return root +} + +/** + * Creates a TokenSet from a string. + * + * The string may contain one or more wildcard characters (*) + * that will allow wildcard matching when intersecting with + * another TokenSet. + * + * @param {string} str - The string to create a TokenSet from. + * @returns {lunr.TokenSet} + */ +lunr.TokenSet.fromString = function (str) { + var node = new lunr.TokenSet, + root = node + + /* + * Iterates through all characters within the passed string + * appending a node for each character. + * + * When a wildcard character is found then a self + * referencing edge is introduced to continually match + * any number of any characters. + */ + for (var i = 0, len = str.length; i < len; i++) { + var char = str[i], + final = (i == len - 1) + + if (char == "*") { + node.edges[char] = node + node.final = final + + } else { + var next = new lunr.TokenSet + next.final = final + + node.edges[char] = next + node = next + } + } + + return root +} + +/** + * Converts this TokenSet into an array of strings + * contained within the TokenSet. + * + * This is not intended to be used on a TokenSet that + * contains wildcards, in these cases the results are + * undefined and are likely to cause an infinite loop. + * + * @returns {string[]} + */ +lunr.TokenSet.prototype.toArray = function () { + var words = [] + + var stack = [{ + prefix: "", + node: this + }] + + while (stack.length) { + var frame = stack.pop(), + edges = Object.keys(frame.node.edges), + len = edges.length + + if (frame.node.final) { + /* In Safari, at this point the prefix is sometimes corrupted, see: + * https://github.com/olivernn/lunr.js/issues/279 Calling any + * String.prototype method forces Safari to "cast" this string to what + * it's supposed to be, fixing the bug. */ + frame.prefix.charAt(0) + words.push(frame.prefix) + } + + for (var i = 0; i < len; i++) { + var edge = edges[i] + + stack.push({ + prefix: frame.prefix.concat(edge), + node: frame.node.edges[edge] + }) + } + } + + return words +} + +/** + * Generates a string representation of a TokenSet. + * + * This is intended to allow TokenSets to be used as keys + * in objects, largely to aid the construction and minimisation + * of a TokenSet. As such it is not designed to be a human + * friendly representation of the TokenSet. + * + * @returns {string} + */ +lunr.TokenSet.prototype.toString = function () { + // NOTE: Using Object.keys here as this.edges is very likely + // to enter 'hash-mode' with many keys being added + // + // avoiding a for-in loop here as it leads to the function + // being de-optimised (at least in V8). From some simple + // benchmarks the performance is comparable, but allowing + // V8 to optimize may mean easy performance wins in the future. + + if (this._str) { + return this._str + } + + var str = this.final ? '1' : '0', + labels = Object.keys(this.edges).sort(), + len = labels.length + + for (var i = 0; i < len; i++) { + var label = labels[i], + node = this.edges[label] + + str = str + label + node.id + } + + return str +} + +/** + * Returns a new TokenSet that is the intersection of + * this TokenSet and the passed TokenSet. + * + * This intersection will take into account any wildcards + * contained within the TokenSet. + * + * @param {lunr.TokenSet} b - An other TokenSet to intersect with. + * @returns {lunr.TokenSet} + */ +lunr.TokenSet.prototype.intersect = function (b) { + var output = new lunr.TokenSet, + frame = undefined + + var stack = [{ + qNode: b, + output: output, + node: this + }] + + while (stack.length) { + frame = stack.pop() + + // NOTE: As with the #toString method, we are using + // Object.keys and a for loop instead of a for-in loop + // as both of these objects enter 'hash' mode, causing + // the function to be de-optimised in V8 + var qEdges = Object.keys(frame.qNode.edges), + qLen = qEdges.length, + nEdges = Object.keys(frame.node.edges), + nLen = nEdges.length + + for (var q = 0; q < qLen; q++) { + var qEdge = qEdges[q] + + for (var n = 0; n < nLen; n++) { + var nEdge = nEdges[n] + + if (nEdge == qEdge || qEdge == '*') { + var node = frame.node.edges[nEdge], + qNode = frame.qNode.edges[qEdge], + final = node.final && qNode.final, + next = undefined + + if (nEdge in frame.output.edges) { + // an edge already exists for this character + // no need to create a new node, just set the finality + // bit unless this node is already final + next = frame.output.edges[nEdge] + next.final = next.final || final + + } else { + // no edge exists yet, must create one + // set the finality bit and insert it + // into the output + next = new lunr.TokenSet + next.final = final + frame.output.edges[nEdge] = next + } + + stack.push({ + qNode: qNode, + output: next, + node: node + }) + } + } + } + } + + return output +} +lunr.TokenSet.Builder = function () { + this.previousWord = "" + this.root = new lunr.TokenSet + this.uncheckedNodes = [] + this.minimizedNodes = {} +} + +lunr.TokenSet.Builder.prototype.insert = function (word) { + var node, + commonPrefix = 0 + + if (word < this.previousWord) { + throw new Error ("Out of order word insertion") + } + + for (var i = 0; i < word.length && i < this.previousWord.length; i++) { + if (word[i] != this.previousWord[i]) break + commonPrefix++ + } + + this.minimize(commonPrefix) + + if (this.uncheckedNodes.length == 0) { + node = this.root + } else { + node = this.uncheckedNodes[this.uncheckedNodes.length - 1].child + } + + for (var i = commonPrefix; i < word.length; i++) { + var nextNode = new lunr.TokenSet, + char = word[i] + + node.edges[char] = nextNode + + this.uncheckedNodes.push({ + parent: node, + char: char, + child: nextNode + }) + + node = nextNode + } + + node.final = true + this.previousWord = word +} + +lunr.TokenSet.Builder.prototype.finish = function () { + this.minimize(0) +} + +lunr.TokenSet.Builder.prototype.minimize = function (downTo) { + for (var i = this.uncheckedNodes.length - 1; i >= downTo; i--) { + var node = this.uncheckedNodes[i], + childKey = node.child.toString() + + if (childKey in this.minimizedNodes) { + node.parent.edges[node.char] = this.minimizedNodes[childKey] + } else { + // Cache the key for this node since + // we know it can't change anymore + node.child._str = childKey + + this.minimizedNodes[childKey] = node.child + } + + this.uncheckedNodes.pop() + } +} +/*! + * lunr.Index + * Copyright (C) 2022 Oliver Nightingale + */ + +/** + * An index contains the built index of all documents and provides a query interface + * to the index. + * + * Usually instances of lunr.Index will not be created using this constructor, instead + * lunr.Builder should be used to construct new indexes, or lunr.Index.load should be + * used to load previously built and serialized indexes. + * + * @constructor + * @param {Object} attrs - The attributes of the built search index. + * @param {Object} attrs.invertedIndex - An index of term/field to document reference. + * @param {Object<string, lunr.Vector>} attrs.fieldVectors - Field vectors + * @param {lunr.TokenSet} attrs.tokenSet - An set of all corpus tokens. + * @param {string[]} attrs.fields - The names of indexed document fields. + * @param {lunr.Pipeline} attrs.pipeline - The pipeline to use for search terms. + */ +lunr.Index = function (attrs) { + this.invertedIndex = attrs.invertedIndex + this.fieldVectors = attrs.fieldVectors + this.tokenSet = attrs.tokenSet + this.fields = attrs.fields + this.pipeline = attrs.pipeline +} + +/** + * A result contains details of a document matching a search query. + * @typedef {Object} lunr.Index~Result + * @property {string} ref - The reference of the document this result represents. + * @property {number} score - A number between 0 and 1 representing how similar this document is to the query. + * @property {lunr.MatchData} matchData - Contains metadata about this match including which term(s) caused the match. + */ + +/** + * Although lunr provides the ability to create queries using lunr.Query, it also provides a simple + * query language which itself is parsed into an instance of lunr.Query. + * + * For programmatically building queries it is advised to directly use lunr.Query, the query language + * is best used for human entered text rather than program generated text. + * + * At its simplest queries can just be a single term, e.g. `hello`, multiple terms are also supported + * and will be combined with OR, e.g `hello world` will match documents that contain either 'hello' + * or 'world', though those that contain both will rank higher in the results. + * + * Wildcards can be included in terms to match one or more unspecified characters, these wildcards can + * be inserted anywhere within the term, and more than one wildcard can exist in a single term. Adding + * wildcards will increase the number of documents that will be found but can also have a negative + * impact on query performance, especially with wildcards at the beginning of a term. + * + * Terms can be restricted to specific fields, e.g. `title:hello`, only documents with the term + * hello in the title field will match this query. Using a field not present in the index will lead + * to an error being thrown. + * + * Modifiers can also be added to terms, lunr supports edit distance and boost modifiers on terms. A term + * boost will make documents matching that term score higher, e.g. `foo^5`. Edit distance is also supported + * to provide fuzzy matching, e.g. 'hello~2' will match documents with hello with an edit distance of 2. + * Avoid large values for edit distance to improve query performance. + * + * Each term also supports a presence modifier. By default a term's presence in document is optional, however + * this can be changed to either required or prohibited. For a term's presence to be required in a document the + * term should be prefixed with a '+', e.g. `+foo bar` is a search for documents that must contain 'foo' and + * optionally contain 'bar'. Conversely a leading '-' sets the terms presence to prohibited, i.e. it must not + * appear in a document, e.g. `-foo bar` is a search for documents that do not contain 'foo' but may contain 'bar'. + * + * To escape special characters the backslash character '\' can be used, this allows searches to include + * characters that would normally be considered modifiers, e.g. `foo\~2` will search for a term "foo~2" instead + * of attempting to apply a boost of 2 to the search term "foo". + * + * @typedef {string} lunr.Index~QueryString + * @example <caption>Simple single term query</caption> + * hello + * @example <caption>Multiple term query</caption> + * hello world + * @example <caption>term scoped to a field</caption> + * title:hello + * @example <caption>term with a boost of 10</caption> + * hello^10 + * @example <caption>term with an edit distance of 2</caption> + * hello~2 + * @example <caption>terms with presence modifiers</caption> + * -foo +bar baz + */ + +/** + * Performs a search against the index using lunr query syntax. + * + * Results will be returned sorted by their score, the most relevant results + * will be returned first. For details on how the score is calculated, please see + * the {@link https://lunrjs.com/guides/searching.html#scoring|guide}. + * + * For more programmatic querying use lunr.Index#query. + * + * @param {lunr.Index~QueryString} queryString - A string containing a lunr query. + * @throws {lunr.QueryParseError} If the passed query string cannot be parsed. + * @returns {lunr.Index~Result[]} + */ +lunr.Index.prototype.search = function (queryString) { + return this.query(function (query) { + var parser = new lunr.QueryParser(queryString, query) + parser.parse() + }) +} + +/** + * A query builder callback provides a query object to be used to express + * the query to perform on the index. + * + * @callback lunr.Index~queryBuilder + * @param {lunr.Query} query - The query object to build up. + * @this lunr.Query + */ + +/** + * Performs a query against the index using the yielded lunr.Query object. + * + * If performing programmatic queries against the index, this method is preferred + * over lunr.Index#search so as to avoid the additional query parsing overhead. + * + * A query object is yielded to the supplied function which should be used to + * express the query to be run against the index. + * + * Note that although this function takes a callback parameter it is _not_ an + * asynchronous operation, the callback is just yielded a query object to be + * customized. + * + * @param {lunr.Index~queryBuilder} fn - A function that is used to build the query. + * @returns {lunr.Index~Result[]} + */ +lunr.Index.prototype.query = function (fn) { + // for each query clause + // * process terms + // * expand terms from token set + // * find matching documents and metadata + // * get document vectors + // * score documents + + var query = new lunr.Query(this.fields), + matchingFields = Object.create(null), + queryVectors = Object.create(null), + termFieldCache = Object.create(null), + requiredMatches = Object.create(null), + prohibitedMatches = Object.create(null) + + /* + * To support field level boosts a query vector is created per + * field. An empty vector is eagerly created to support negated + * queries. + */ + for (var i = 0; i < this.fields.length; i++) { + queryVectors[this.fields[i]] = new lunr.Vector + } + + fn.call(query, query) + + for (var i = 0; i < query.clauses.length; i++) { + /* + * Unless the pipeline has been disabled for this term, which is + * the case for terms with wildcards, we need to pass the clause + * term through the search pipeline. A pipeline returns an array + * of processed terms. Pipeline functions may expand the passed + * term, which means we may end up performing multiple index lookups + * for a single query term. + */ + var clause = query.clauses[i], + terms = null, + clauseMatches = lunr.Set.empty + + if (clause.usePipeline) { + terms = this.pipeline.runString(clause.term, { + fields: clause.fields + }) + } else { + terms = [clause.term] + } + + for (var m = 0; m < terms.length; m++) { + var term = terms[m] + + /* + * Each term returned from the pipeline needs to use the same query + * clause object, e.g. the same boost and or edit distance. The + * simplest way to do this is to re-use the clause object but mutate + * its term property. + */ + clause.term = term + + /* + * From the term in the clause we create a token set which will then + * be used to intersect the indexes token set to get a list of terms + * to lookup in the inverted index + */ + var termTokenSet = lunr.TokenSet.fromClause(clause), + expandedTerms = this.tokenSet.intersect(termTokenSet).toArray() + + /* + * If a term marked as required does not exist in the tokenSet it is + * impossible for the search to return any matches. We set all the field + * scoped required matches set to empty and stop examining any further + * clauses. + */ + if (expandedTerms.length === 0 && clause.presence === lunr.Query.presence.REQUIRED) { + for (var k = 0; k < clause.fields.length; k++) { + var field = clause.fields[k] + requiredMatches[field] = lunr.Set.empty + } + + break + } + + for (var j = 0; j < expandedTerms.length; j++) { + /* + * For each term get the posting and termIndex, this is required for + * building the query vector. + */ + var expandedTerm = expandedTerms[j], + posting = this.invertedIndex[expandedTerm], + termIndex = posting._index + + for (var k = 0; k < clause.fields.length; k++) { + /* + * For each field that this query term is scoped by (by default + * all fields are in scope) we need to get all the document refs + * that have this term in that field. + * + * The posting is the entry in the invertedIndex for the matching + * term from above. + */ + var field = clause.fields[k], + fieldPosting = posting[field], + matchingDocumentRefs = Object.keys(fieldPosting), + termField = expandedTerm + "/" + field, + matchingDocumentsSet = new lunr.Set(matchingDocumentRefs) + + /* + * if the presence of this term is required ensure that the matching + * documents are added to the set of required matches for this clause. + * + */ + if (clause.presence == lunr.Query.presence.REQUIRED) { + clauseMatches = clauseMatches.union(matchingDocumentsSet) + + if (requiredMatches[field] === undefined) { + requiredMatches[field] = lunr.Set.complete + } + } + + /* + * if the presence of this term is prohibited ensure that the matching + * documents are added to the set of prohibited matches for this field, + * creating that set if it does not yet exist. + */ + if (clause.presence == lunr.Query.presence.PROHIBITED) { + if (prohibitedMatches[field] === undefined) { + prohibitedMatches[field] = lunr.Set.empty + } + + prohibitedMatches[field] = prohibitedMatches[field].union(matchingDocumentsSet) + + /* + * Prohibited matches should not be part of the query vector used for + * similarity scoring and no metadata should be extracted so we continue + * to the next field + */ + continue + } + + /* + * The query field vector is populated using the termIndex found for + * the term and a unit value with the appropriate boost applied. + * Using upsert because there could already be an entry in the vector + * for the term we are working with. In that case we just add the scores + * together. + */ + queryVectors[field].upsert(termIndex, clause.boost, function (a, b) { return a + b }) + + /** + * If we've already seen this term, field combo then we've already collected + * the matching documents and metadata, no need to go through all that again + */ + if (termFieldCache[termField]) { + continue + } + + for (var l = 0; l < matchingDocumentRefs.length; l++) { + /* + * All metadata for this term/field/document triple + * are then extracted and collected into an instance + * of lunr.MatchData ready to be returned in the query + * results + */ + var matchingDocumentRef = matchingDocumentRefs[l], + matchingFieldRef = new lunr.FieldRef (matchingDocumentRef, field), + metadata = fieldPosting[matchingDocumentRef], + fieldMatch + + if ((fieldMatch = matchingFields[matchingFieldRef]) === undefined) { + matchingFields[matchingFieldRef] = new lunr.MatchData (expandedTerm, field, metadata) + } else { + fieldMatch.add(expandedTerm, field, metadata) + } + + } + + termFieldCache[termField] = true + } + } + } + + /** + * If the presence was required we need to update the requiredMatches field sets. + * We do this after all fields for the term have collected their matches because + * the clause terms presence is required in _any_ of the fields not _all_ of the + * fields. + */ + if (clause.presence === lunr.Query.presence.REQUIRED) { + for (var k = 0; k < clause.fields.length; k++) { + var field = clause.fields[k] + requiredMatches[field] = requiredMatches[field].intersect(clauseMatches) + } + } + } + + /** + * Need to combine the field scoped required and prohibited + * matching documents into a global set of required and prohibited + * matches + */ + var allRequiredMatches = lunr.Set.complete, + allProhibitedMatches = lunr.Set.empty + + for (var i = 0; i < this.fields.length; i++) { + var field = this.fields[i] + + if (requiredMatches[field]) { + allRequiredMatches = allRequiredMatches.intersect(requiredMatches[field]) + } + + if (prohibitedMatches[field]) { + allProhibitedMatches = allProhibitedMatches.union(prohibitedMatches[field]) + } + } + + var matchingFieldRefs = Object.keys(matchingFields), + results = [], + matches = Object.create(null) + + /* + * If the query is negated (contains only prohibited terms) + * we need to get _all_ fieldRefs currently existing in the + * index. This is only done when we know that the query is + * entirely prohibited terms to avoid any cost of getting all + * fieldRefs unnecessarily. + * + * Additionally, blank MatchData must be created to correctly + * populate the results. + */ + if (query.isNegated()) { + matchingFieldRefs = Object.keys(this.fieldVectors) + + for (var i = 0; i < matchingFieldRefs.length; i++) { + var matchingFieldRef = matchingFieldRefs[i] + var fieldRef = lunr.FieldRef.fromString(matchingFieldRef) + matchingFields[matchingFieldRef] = new lunr.MatchData + } + } + + for (var i = 0; i < matchingFieldRefs.length; i++) { + /* + * Currently we have document fields that match the query, but we + * need to return documents. The matchData and scores are combined + * from multiple fields belonging to the same document. + * + * Scores are calculated by field, using the query vectors created + * above, and combined into a final document score using addition. + */ + var fieldRef = lunr.FieldRef.fromString(matchingFieldRefs[i]), + docRef = fieldRef.docRef + + if (!allRequiredMatches.contains(docRef)) { + continue + } + + if (allProhibitedMatches.contains(docRef)) { + continue + } + + var fieldVector = this.fieldVectors[fieldRef], + score = queryVectors[fieldRef.fieldName].similarity(fieldVector), + docMatch + + if ((docMatch = matches[docRef]) !== undefined) { + docMatch.score += score + docMatch.matchData.combine(matchingFields[fieldRef]) + } else { + var match = { + ref: docRef, + score: score, + matchData: matchingFields[fieldRef] + } + matches[docRef] = match + results.push(match) + } + } + + /* + * Sort the results objects by score, highest first. + */ + return results.sort(function (a, b) { + return b.score - a.score + }) +} + +/** + * Prepares the index for JSON serialization. + * + * The schema for this JSON blob will be described in a + * separate JSON schema file. + * + * @returns {Object} + */ +lunr.Index.prototype.toJSON = function () { + var invertedIndex = Object.keys(this.invertedIndex) + .sort() + .map(function (term) { + return [term, this.invertedIndex[term]] + }, this) + + var fieldVectors = Object.keys(this.fieldVectors) + .map(function (ref) { + return [ref, this.fieldVectors[ref].toJSON()] + }, this) + + return { + version: lunr.version, + fields: this.fields, + fieldVectors: fieldVectors, + invertedIndex: invertedIndex, + pipeline: this.pipeline.toJSON() + } +} + +/** + * Loads a previously serialized lunr.Index + * + * @param {Object} serializedIndex - A previously serialized lunr.Index + * @returns {lunr.Index} + */ +lunr.Index.load = function (serializedIndex) { + var attrs = {}, + fieldVectors = {}, + serializedVectors = serializedIndex.fieldVectors, + invertedIndex = Object.create(null), + serializedInvertedIndex = serializedIndex.invertedIndex, + tokenSetBuilder = new lunr.TokenSet.Builder, + pipeline = lunr.Pipeline.load(serializedIndex.pipeline) + + if (serializedIndex.version != lunr.version) { + lunr.utils.warn("Version mismatch when loading serialised index. Current version of lunr '" + lunr.version + "' does not match serialized index '" + serializedIndex.version + "'") + } + + for (var i = 0; i < serializedVectors.length; i++) { + var tuple = serializedVectors[i], + ref = tuple[0], + elements = tuple[1] + + fieldVectors[ref] = new lunr.Vector(elements) + } + + for (var i = 0; i < serializedInvertedIndex.length; i++) { + var tuple = serializedInvertedIndex[i], + term = tuple[0], + posting = tuple[1] + + tokenSetBuilder.insert(term) + invertedIndex[term] = posting + } + + tokenSetBuilder.finish() + + attrs.fields = serializedIndex.fields + + attrs.fieldVectors = fieldVectors + attrs.invertedIndex = invertedIndex + attrs.tokenSet = tokenSetBuilder.root + attrs.pipeline = pipeline + + return new lunr.Index(attrs) +} +/*! + * lunr.Builder + * Copyright (C) 2022 Oliver Nightingale + */ + +/** + * lunr.Builder performs indexing on a set of documents and + * returns instances of lunr.Index ready for querying. + * + * All configuration of the index is done via the builder, the + * fields to index, the document reference, the text processing + * pipeline and document scoring parameters are all set on the + * builder before indexing. + * + * @constructor + * @property {string} _ref - Internal reference to the document reference field. + * @property {string[]} _fields - Internal reference to the document fields to index. + * @property {object} invertedIndex - The inverted index maps terms to document fields. + * @property {object} documentTermFrequencies - Keeps track of document term frequencies. + * @property {object} documentLengths - Keeps track of the length of documents added to the index. + * @property {lunr.tokenizer} tokenizer - Function for splitting strings into tokens for indexing. + * @property {lunr.Pipeline} pipeline - The pipeline performs text processing on tokens before indexing. + * @property {lunr.Pipeline} searchPipeline - A pipeline for processing search terms before querying the index. + * @property {number} documentCount - Keeps track of the total number of documents indexed. + * @property {number} _b - A parameter to control field length normalization, setting this to 0 disabled normalization, 1 fully normalizes field lengths, the default value is 0.75. + * @property {number} _k1 - A parameter to control how quickly an increase in term frequency results in term frequency saturation, the default value is 1.2. + * @property {number} termIndex - A counter incremented for each unique term, used to identify a terms position in the vector space. + * @property {array} metadataWhitelist - A list of metadata keys that have been whitelisted for entry in the index. + */ +lunr.Builder = function () { + this._ref = "id" + this._fields = Object.create(null) + this._documents = Object.create(null) + this.invertedIndex = Object.create(null) + this.fieldTermFrequencies = {} + this.fieldLengths = {} + this.tokenizer = lunr.tokenizer + this.pipeline = new lunr.Pipeline + this.searchPipeline = new lunr.Pipeline + this.documentCount = 0 + this._b = 0.75 + this._k1 = 1.2 + this.termIndex = 0 + this.metadataWhitelist = [] +} + +/** + * Sets the document field used as the document reference. Every document must have this field. + * The type of this field in the document should be a string, if it is not a string it will be + * coerced into a string by calling toString. + * + * The default ref is 'id'. + * + * The ref should _not_ be changed during indexing, it should be set before any documents are + * added to the index. Changing it during indexing can lead to inconsistent results. + * + * @param {string} ref - The name of the reference field in the document. + */ +lunr.Builder.prototype.ref = function (ref) { + this._ref = ref +} + +/** + * A function that is used to extract a field from a document. + * + * Lunr expects a field to be at the top level of a document, if however the field + * is deeply nested within a document an extractor function can be used to extract + * the right field for indexing. + * + * @callback fieldExtractor + * @param {object} doc - The document being added to the index. + * @returns {?(string|object|object[])} obj - The object that will be indexed for this field. + * @example <caption>Extracting a nested field</caption> + * function (doc) { return doc.nested.field } + */ + +/** + * Adds a field to the list of document fields that will be indexed. Every document being + * indexed should have this field. Null values for this field in indexed documents will + * not cause errors but will limit the chance of that document being retrieved by searches. + * + * All fields should be added before adding documents to the index. Adding fields after + * a document has been indexed will have no effect on already indexed documents. + * + * Fields can be boosted at build time. This allows terms within that field to have more + * importance when ranking search results. Use a field boost to specify that matches within + * one field are more important than other fields. + * + * @param {string} fieldName - The name of a field to index in all documents. + * @param {object} attributes - Optional attributes associated with this field. + * @param {number} [attributes.boost=1] - Boost applied to all terms within this field. + * @param {fieldExtractor} [attributes.extractor] - Function to extract a field from a document. + * @throws {RangeError} fieldName cannot contain unsupported characters '/' + */ +lunr.Builder.prototype.field = function (fieldName, attributes) { + if (/\//.test(fieldName)) { + throw new RangeError ("Field '" + fieldName + "' contains illegal character '/'") + } + + this._fields[fieldName] = attributes || {} +} + +/** + * A parameter to tune the amount of field length normalisation that is applied when + * calculating relevance scores. A value of 0 will completely disable any normalisation + * and a value of 1 will fully normalise field lengths. The default is 0.75. Values of b + * will be clamped to the range 0 - 1. + * + * @param {number} number - The value to set for this tuning parameter. + */ +lunr.Builder.prototype.b = function (number) { + if (number < 0) { + this._b = 0 + } else if (number > 1) { + this._b = 1 + } else { + this._b = number + } +} + +/** + * A parameter that controls the speed at which a rise in term frequency results in term + * frequency saturation. The default value is 1.2. Setting this to a higher value will give + * slower saturation levels, a lower value will result in quicker saturation. + * + * @param {number} number - The value to set for this tuning parameter. + */ +lunr.Builder.prototype.k1 = function (number) { + this._k1 = number +} + +/** + * Adds a document to the index. + * + * Before adding fields to the index the index should have been fully setup, with the document + * ref and all fields to index already having been specified. + * + * The document must have a field name as specified by the ref (by default this is 'id') and + * it should have all fields defined for indexing, though null or undefined values will not + * cause errors. + * + * Entire documents can be boosted at build time. Applying a boost to a document indicates that + * this document should rank higher in search results than other documents. + * + * @param {object} doc - The document to add to the index. + * @param {object} attributes - Optional attributes associated with this document. + * @param {number} [attributes.boost=1] - Boost applied to all terms within this document. + */ +lunr.Builder.prototype.add = function (doc, attributes) { + var docRef = doc[this._ref], + fields = Object.keys(this._fields) + + this._documents[docRef] = attributes || {} + this.documentCount += 1 + + for (var i = 0; i < fields.length; i++) { + var fieldName = fields[i], + extractor = this._fields[fieldName].extractor, + field = extractor ? extractor(doc) : doc[fieldName], + tokens = this.tokenizer(field, { + fields: [fieldName] + }), + terms = this.pipeline.run(tokens), + fieldRef = new lunr.FieldRef (docRef, fieldName), + fieldTerms = Object.create(null) + + this.fieldTermFrequencies[fieldRef] = fieldTerms + this.fieldLengths[fieldRef] = 0 + + // store the length of this field for this document + this.fieldLengths[fieldRef] += terms.length + + // calculate term frequencies for this field + for (var j = 0; j < terms.length; j++) { + var term = terms[j] + + if (fieldTerms[term] == undefined) { + fieldTerms[term] = 0 + } + + fieldTerms[term] += 1 + + // add to inverted index + // create an initial posting if one doesn't exist + if (this.invertedIndex[term] == undefined) { + var posting = Object.create(null) + posting["_index"] = this.termIndex + this.termIndex += 1 + + for (var k = 0; k < fields.length; k++) { + posting[fields[k]] = Object.create(null) + } + + this.invertedIndex[term] = posting + } + + // add an entry for this term/fieldName/docRef to the invertedIndex + if (this.invertedIndex[term][fieldName][docRef] == undefined) { + this.invertedIndex[term][fieldName][docRef] = Object.create(null) + } + + // store all whitelisted metadata about this token in the + // inverted index + for (var l = 0; l < this.metadataWhitelist.length; l++) { + var metadataKey = this.metadataWhitelist[l], + metadata = term.metadata[metadataKey] + + if (this.invertedIndex[term][fieldName][docRef][metadataKey] == undefined) { + this.invertedIndex[term][fieldName][docRef][metadataKey] = [] + } + + this.invertedIndex[term][fieldName][docRef][metadataKey].push(metadata) + } + } + + } +} + +/** + * Calculates the average document length for this index + * + * @private + */ +lunr.Builder.prototype.calculateAverageFieldLengths = function () { + + var fieldRefs = Object.keys(this.fieldLengths), + numberOfFields = fieldRefs.length, + accumulator = {}, + documentsWithField = {} + + for (var i = 0; i < numberOfFields; i++) { + var fieldRef = lunr.FieldRef.fromString(fieldRefs[i]), + field = fieldRef.fieldName + + documentsWithField[field] || (documentsWithField[field] = 0) + documentsWithField[field] += 1 + + accumulator[field] || (accumulator[field] = 0) + accumulator[field] += this.fieldLengths[fieldRef] + } + + var fields = Object.keys(this._fields) + + for (var i = 0; i < fields.length; i++) { + var fieldName = fields[i] + accumulator[fieldName] = accumulator[fieldName] / documentsWithField[fieldName] + } + + this.averageFieldLength = accumulator +} + +/** + * Builds a vector space model of every document using lunr.Vector + * + * @private + */ +lunr.Builder.prototype.createFieldVectors = function () { + var fieldVectors = {}, + fieldRefs = Object.keys(this.fieldTermFrequencies), + fieldRefsLength = fieldRefs.length, + termIdfCache = Object.create(null) + + for (var i = 0; i < fieldRefsLength; i++) { + var fieldRef = lunr.FieldRef.fromString(fieldRefs[i]), + fieldName = fieldRef.fieldName, + fieldLength = this.fieldLengths[fieldRef], + fieldVector = new lunr.Vector, + termFrequencies = this.fieldTermFrequencies[fieldRef], + terms = Object.keys(termFrequencies), + termsLength = terms.length + + + var fieldBoost = this._fields[fieldName].boost || 1, + docBoost = this._documents[fieldRef.docRef].boost || 1 + + for (var j = 0; j < termsLength; j++) { + var term = terms[j], + tf = termFrequencies[term], + termIndex = this.invertedIndex[term]._index, + idf, score, scoreWithPrecision + + if (termIdfCache[term] === undefined) { + idf = lunr.idf(this.invertedIndex[term], this.documentCount) + termIdfCache[term] = idf + } else { + idf = termIdfCache[term] + } + + score = idf * ((this._k1 + 1) * tf) / (this._k1 * (1 - this._b + this._b * (fieldLength / this.averageFieldLength[fieldName])) + tf) + score *= fieldBoost + score *= docBoost + scoreWithPrecision = Math.round(score * 1000) / 1000 + // Converts 1.23456789 to 1.234. + // Reducing the precision so that the vectors take up less + // space when serialised. Doing it now so that they behave + // the same before and after serialisation. Also, this is + // the fastest approach to reducing a number's precision in + // JavaScript. + + fieldVector.insert(termIndex, scoreWithPrecision) + } + + fieldVectors[fieldRef] = fieldVector + } + + this.fieldVectors = fieldVectors +} + +/** + * Creates a token set of all tokens in the index using lunr.TokenSet + * + * @private + */ +lunr.Builder.prototype.createTokenSet = function () { + this.tokenSet = lunr.TokenSet.fromArray( + Object.keys(this.invertedIndex).sort() + ) +} + +/** + * Builds the index, creating an instance of lunr.Index. + * + * This completes the indexing process and should only be called + * once all documents have been added to the index. + * + * @returns {lunr.Index} + */ +lunr.Builder.prototype.build = function () { + this.calculateAverageFieldLengths() + this.createFieldVectors() + this.createTokenSet() + + return new lunr.Index({ + invertedIndex: this.invertedIndex, + fieldVectors: this.fieldVectors, + tokenSet: this.tokenSet, + fields: Object.keys(this._fields), + pipeline: this.searchPipeline + }) +} + +/** + * Applies a plugin to the index builder. + * + * A plugin is a function that is called with the index builder as its context. + * Plugins can be used to customise or extend the behaviour of the index + * in some way. A plugin is just a function, that encapsulated the custom + * behaviour that should be applied when building the index. + * + * The plugin function will be called with the index builder as its argument, additional + * arguments can also be passed when calling use. The function will be called + * with the index builder as its context. + * + * @param {Function} plugin The plugin to apply. + */ +lunr.Builder.prototype.use = function (fn) { + var args = Array.prototype.slice.call(arguments, 1) + args.unshift(this) + fn.apply(this, args) +} +/** + * Contains and collects metadata about a matching document. + * A single instance of lunr.MatchData is returned as part of every + * lunr.Index~Result. + * + * @constructor + * @param {string} term - The term this match data is associated with + * @param {string} field - The field in which the term was found + * @param {object} metadata - The metadata recorded about this term in this field + * @property {object} metadata - A cloned collection of metadata associated with this document. + * @see {@link lunr.Index~Result} + */ +lunr.MatchData = function (term, field, metadata) { + var clonedMetadata = Object.create(null), + metadataKeys = Object.keys(metadata || {}) + + // Cloning the metadata to prevent the original + // being mutated during match data combination. + // Metadata is kept in an array within the inverted + // index so cloning the data can be done with + // Array#slice + for (var i = 0; i < metadataKeys.length; i++) { + var key = metadataKeys[i] + clonedMetadata[key] = metadata[key].slice() + } + + this.metadata = Object.create(null) + + if (term !== undefined) { + this.metadata[term] = Object.create(null) + this.metadata[term][field] = clonedMetadata + } +} + +/** + * An instance of lunr.MatchData will be created for every term that matches a + * document. However only one instance is required in a lunr.Index~Result. This + * method combines metadata from another instance of lunr.MatchData with this + * objects metadata. + * + * @param {lunr.MatchData} otherMatchData - Another instance of match data to merge with this one. + * @see {@link lunr.Index~Result} + */ +lunr.MatchData.prototype.combine = function (otherMatchData) { + var terms = Object.keys(otherMatchData.metadata) + + for (var i = 0; i < terms.length; i++) { + var term = terms[i], + fields = Object.keys(otherMatchData.metadata[term]) + + if (this.metadata[term] == undefined) { + this.metadata[term] = Object.create(null) + } + + for (var j = 0; j < fields.length; j++) { + var field = fields[j], + keys = Object.keys(otherMatchData.metadata[term][field]) + + if (this.metadata[term][field] == undefined) { + this.metadata[term][field] = Object.create(null) + } + + for (var k = 0; k < keys.length; k++) { + var key = keys[k] + + if (this.metadata[term][field][key] == undefined) { + this.metadata[term][field][key] = otherMatchData.metadata[term][field][key] + } else { + this.metadata[term][field][key] = this.metadata[term][field][key].concat(otherMatchData.metadata[term][field][key]) + } + + } + } + } +} + +/** + * Add metadata for a term/field pair to this instance of match data. + * + * @param {string} term - The term this match data is associated with + * @param {string} field - The field in which the term was found + * @param {object} metadata - The metadata recorded about this term in this field + */ +lunr.MatchData.prototype.add = function (term, field, metadata) { + if (!(term in this.metadata)) { + this.metadata[term] = Object.create(null) + this.metadata[term][field] = metadata + return + } + + if (!(field in this.metadata[term])) { + this.metadata[term][field] = metadata + return + } + + var metadataKeys = Object.keys(metadata) + + for (var i = 0; i < metadataKeys.length; i++) { + var key = metadataKeys[i] + + if (key in this.metadata[term][field]) { + this.metadata[term][field][key] = this.metadata[term][field][key].concat(metadata[key]) + } else { + this.metadata[term][field][key] = metadata[key] + } + } +} +/** + * A lunr.Query provides a programmatic way of defining queries to be performed + * against a {@link lunr.Index}. + * + * Prefer constructing a lunr.Query using the {@link lunr.Index#query} method + * so the query object is pre-initialized with the right index fields. + * + * @constructor + * @property {lunr.Query~Clause[]} clauses - An array of query clauses. + * @property {string[]} allFields - An array of all available fields in a lunr.Index. + */ +lunr.Query = function (allFields) { + this.clauses = [] + this.allFields = allFields +} + +/** + * Constants for indicating what kind of automatic wildcard insertion will be used when constructing a query clause. + * + * This allows wildcards to be added to the beginning and end of a term without having to manually do any string + * concatenation. + * + * The wildcard constants can be bitwise combined to select both leading and trailing wildcards. + * + * @constant + * @default + * @property {number} wildcard.NONE - The term will have no wildcards inserted, this is the default behaviour + * @property {number} wildcard.LEADING - Prepend the term with a wildcard, unless a leading wildcard already exists + * @property {number} wildcard.TRAILING - Append a wildcard to the term, unless a trailing wildcard already exists + * @see lunr.Query~Clause + * @see lunr.Query#clause + * @see lunr.Query#term + * @example <caption>query term with trailing wildcard</caption> + * query.term('foo', { wildcard: lunr.Query.wildcard.TRAILING }) + * @example <caption>query term with leading and trailing wildcard</caption> + * query.term('foo', { + * wildcard: lunr.Query.wildcard.LEADING | lunr.Query.wildcard.TRAILING + * }) + */ + +lunr.Query.wildcard = new String ("*") +lunr.Query.wildcard.NONE = 0 +lunr.Query.wildcard.LEADING = 1 +lunr.Query.wildcard.TRAILING = 2 + +/** + * Constants for indicating what kind of presence a term must have in matching documents. + * + * @constant + * @enum {number} + * @see lunr.Query~Clause + * @see lunr.Query#clause + * @see lunr.Query#term + * @example <caption>query term with required presence</caption> + * query.term('foo', { presence: lunr.Query.presence.REQUIRED }) + */ +lunr.Query.presence = { + /** + * Term's presence in a document is optional, this is the default value. + */ + OPTIONAL: 1, + + /** + * Term's presence in a document is required, documents that do not contain + * this term will not be returned. + */ + REQUIRED: 2, + + /** + * Term's presence in a document is prohibited, documents that do contain + * this term will not be returned. + */ + PROHIBITED: 3 +} + +/** + * A single clause in a {@link lunr.Query} contains a term and details on how to + * match that term against a {@link lunr.Index}. + * + * @typedef {Object} lunr.Query~Clause + * @property {string[]} fields - The fields in an index this clause should be matched against. + * @property {number} [boost=1] - Any boost that should be applied when matching this clause. + * @property {number} [editDistance] - Whether the term should have fuzzy matching applied, and how fuzzy the match should be. + * @property {boolean} [usePipeline] - Whether the term should be passed through the search pipeline. + * @property {number} [wildcard=lunr.Query.wildcard.NONE] - Whether the term should have wildcards appended or prepended. + * @property {number} [presence=lunr.Query.presence.OPTIONAL] - The terms presence in any matching documents. + */ + +/** + * Adds a {@link lunr.Query~Clause} to this query. + * + * Unless the clause contains the fields to be matched all fields will be matched. In addition + * a default boost of 1 is applied to the clause. + * + * @param {lunr.Query~Clause} clause - The clause to add to this query. + * @see lunr.Query~Clause + * @returns {lunr.Query} + */ +lunr.Query.prototype.clause = function (clause) { + if (!('fields' in clause)) { + clause.fields = this.allFields + } + + if (!('boost' in clause)) { + clause.boost = 1 + } + + if (!('usePipeline' in clause)) { + clause.usePipeline = true + } + + if (!('wildcard' in clause)) { + clause.wildcard = lunr.Query.wildcard.NONE + } + + if ((clause.wildcard & lunr.Query.wildcard.LEADING) && (clause.term.charAt(0) != lunr.Query.wildcard)) { + clause.term = "*" + clause.term + } + + if ((clause.wildcard & lunr.Query.wildcard.TRAILING) && (clause.term.slice(-1) != lunr.Query.wildcard)) { + clause.term = "" + clause.term + "*" + } + + if (!('presence' in clause)) { + clause.presence = lunr.Query.presence.OPTIONAL + } + + this.clauses.push(clause) + + return this +} + +/** + * A negated query is one in which every clause has a presence of + * prohibited. These queries require some special processing to return + * the expected results. + * + * @returns boolean + */ +lunr.Query.prototype.isNegated = function () { + for (var i = 0; i < this.clauses.length; i++) { + if (this.clauses[i].presence != lunr.Query.presence.PROHIBITED) { + return false + } + } + + return true +} + +/** + * Adds a term to the current query, under the covers this will create a {@link lunr.Query~Clause} + * to the list of clauses that make up this query. + * + * The term is used as is, i.e. no tokenization will be performed by this method. Instead conversion + * to a token or token-like string should be done before calling this method. + * + * The term will be converted to a string by calling `toString`. Multiple terms can be passed as an + * array, each term in the array will share the same options. + * + * @param {object|object[]} term - The term(s) to add to the query. + * @param {object} [options] - Any additional properties to add to the query clause. + * @returns {lunr.Query} + * @see lunr.Query#clause + * @see lunr.Query~Clause + * @example <caption>adding a single term to a query</caption> + * query.term("foo") + * @example <caption>adding a single term to a query and specifying search fields, term boost and automatic trailing wildcard</caption> + * query.term("foo", { + * fields: ["title"], + * boost: 10, + * wildcard: lunr.Query.wildcard.TRAILING + * }) + * @example <caption>using lunr.tokenizer to convert a string to tokens before using them as terms</caption> + * query.term(lunr.tokenizer("foo bar")) + */ +lunr.Query.prototype.term = function (term, options) { + if (Array.isArray(term)) { + term.forEach(function (t) { this.term(t, lunr.utils.clone(options)) }, this) + return this + } + + var clause = options || {} + clause.term = term.toString() + + this.clause(clause) + + return this +} +lunr.QueryParseError = function (message, start, end) { + this.name = "QueryParseError" + this.message = message + this.start = start + this.end = end +} + +lunr.QueryParseError.prototype = new Error +lunr.QueryLexer = function (str) { + this.lexemes = [] + this.str = str + this.length = str.length + this.pos = 0 + this.start = 0 + this.escapeCharPositions = [] +} + +lunr.QueryLexer.prototype.run = function () { + var state = lunr.QueryLexer.lexText + + while (state) { + state = state(this) + } +} + +lunr.QueryLexer.prototype.sliceString = function () { + var subSlices = [], + sliceStart = this.start, + sliceEnd = this.pos + + for (var i = 0; i < this.escapeCharPositions.length; i++) { + sliceEnd = this.escapeCharPositions[i] + subSlices.push(this.str.slice(sliceStart, sliceEnd)) + sliceStart = sliceEnd + 1 + } + + subSlices.push(this.str.slice(sliceStart, this.pos)) + this.escapeCharPositions.length = 0 + + return subSlices.join('') +} + +lunr.QueryLexer.prototype.emit = function (type) { + this.lexemes.push({ + type: type, + str: this.sliceString(), + start: this.start, + end: this.pos + }) + + this.start = this.pos +} + +lunr.QueryLexer.prototype.escapeCharacter = function () { + this.escapeCharPositions.push(this.pos - 1) + this.pos += 1 +} + +lunr.QueryLexer.prototype.next = function () { + if (this.pos >= this.length) { + return lunr.QueryLexer.EOS + } + + var char = this.str.charAt(this.pos) + this.pos += 1 + return char +} + +lunr.QueryLexer.prototype.width = function () { + return this.pos - this.start +} + +lunr.QueryLexer.prototype.ignore = function () { + if (this.start == this.pos) { + this.pos += 1 + } + + this.start = this.pos +} + +lunr.QueryLexer.prototype.backup = function () { + this.pos -= 1 +} + +lunr.QueryLexer.prototype.acceptDigitRun = function () { + var char, charCode + + do { + char = this.next() + charCode = char.charCodeAt(0) + } while (charCode > 47 && charCode < 58) + + if (char != lunr.QueryLexer.EOS) { + this.backup() + } +} + +lunr.QueryLexer.prototype.more = function () { + return this.pos < this.length +} + +lunr.QueryLexer.EOS = 'EOS' +lunr.QueryLexer.FIELD = 'FIELD' +lunr.QueryLexer.TERM = 'TERM' +lunr.QueryLexer.EDIT_DISTANCE = 'EDIT_DISTANCE' +lunr.QueryLexer.BOOST = 'BOOST' +lunr.QueryLexer.PRESENCE = 'PRESENCE' + +lunr.QueryLexer.lexField = function (lexer) { + lexer.backup() + lexer.emit(lunr.QueryLexer.FIELD) + lexer.ignore() + return lunr.QueryLexer.lexText +} + +lunr.QueryLexer.lexTerm = function (lexer) { + if (lexer.width() > 1) { + lexer.backup() + lexer.emit(lunr.QueryLexer.TERM) + } + + lexer.ignore() + + if (lexer.more()) { + return lunr.QueryLexer.lexText + } +} + +lunr.QueryLexer.lexEditDistance = function (lexer) { + lexer.ignore() + lexer.acceptDigitRun() + lexer.emit(lunr.QueryLexer.EDIT_DISTANCE) + return lunr.QueryLexer.lexText +} + +lunr.QueryLexer.lexBoost = function (lexer) { + lexer.ignore() + lexer.acceptDigitRun() + lexer.emit(lunr.QueryLexer.BOOST) + return lunr.QueryLexer.lexText +} + +lunr.QueryLexer.lexEOS = function (lexer) { + if (lexer.width() > 0) { + lexer.emit(lunr.QueryLexer.TERM) + } +} + +// This matches the separator used when tokenising fields +// within a document. These should match otherwise it is +// not possible to search for some tokens within a document. +// +// It is possible for the user to change the separator on the +// tokenizer so it _might_ clash with any other of the special +// characters already used within the search string, e.g. :. +// +// This means that it is possible to change the separator in +// such a way that makes some words unsearchable using a search +// string. +lunr.QueryLexer.termSeparator = lunr.tokenizer.separator + +lunr.QueryLexer.lexText = function (lexer) { + while (true) { + var char = lexer.next() + + if (char == lunr.QueryLexer.EOS) { + return lunr.QueryLexer.lexEOS + } + + // Escape character is '\' + if (char.charCodeAt(0) == 92) { + lexer.escapeCharacter() + continue + } + + if (char == ":") { + return lunr.QueryLexer.lexField + } + + if (char == "~") { + lexer.backup() + if (lexer.width() > 0) { + lexer.emit(lunr.QueryLexer.TERM) + } + return lunr.QueryLexer.lexEditDistance + } + + if (char == "^") { + lexer.backup() + if (lexer.width() > 0) { + lexer.emit(lunr.QueryLexer.TERM) + } + return lunr.QueryLexer.lexBoost + } + + // "+" indicates term presence is required + // checking for length to ensure that only + // leading "+" are considered + if (char == "+" && lexer.width() === 1) { + lexer.emit(lunr.QueryLexer.PRESENCE) + return lunr.QueryLexer.lexText + } + + // "-" indicates term presence is prohibited + // checking for length to ensure that only + // leading "-" are considered + if (char == "-" && lexer.width() === 1) { + lexer.emit(lunr.QueryLexer.PRESENCE) + return lunr.QueryLexer.lexText + } + + if (char.match(lunr.QueryLexer.termSeparator)) { + return lunr.QueryLexer.lexTerm + } + } +} + +lunr.QueryParser = function (str, query) { + this.lexer = new lunr.QueryLexer (str) + this.query = query + this.currentClause = {} + this.lexemeIdx = 0 +} + +lunr.QueryParser.prototype.parse = function () { + this.lexer.run() + this.lexemes = this.lexer.lexemes + + var state = lunr.QueryParser.parseClause + + while (state) { + state = state(this) + } + + return this.query +} + +lunr.QueryParser.prototype.peekLexeme = function () { + return this.lexemes[this.lexemeIdx] +} + +lunr.QueryParser.prototype.consumeLexeme = function () { + var lexeme = this.peekLexeme() + this.lexemeIdx += 1 + return lexeme +} + +lunr.QueryParser.prototype.nextClause = function () { + var completedClause = this.currentClause + this.query.clause(completedClause) + this.currentClause = {} +} + +lunr.QueryParser.parseClause = function (parser) { + var lexeme = parser.peekLexeme() + + if (lexeme == undefined) { + return + } + + switch (lexeme.type) { + case lunr.QueryLexer.PRESENCE: + return lunr.QueryParser.parsePresence + case lunr.QueryLexer.FIELD: + return lunr.QueryParser.parseField + case lunr.QueryLexer.TERM: + return lunr.QueryParser.parseTerm + default: + var errorMessage = "expected either a field or a term, found " + lexeme.type + + if (lexeme.str.length >= 1) { + errorMessage += " with value '" + lexeme.str + "'" + } + + throw new lunr.QueryParseError (errorMessage, lexeme.start, lexeme.end) + } +} + +lunr.QueryParser.parsePresence = function (parser) { + var lexeme = parser.consumeLexeme() + + if (lexeme == undefined) { + return + } + + switch (lexeme.str) { + case "-": + parser.currentClause.presence = lunr.Query.presence.PROHIBITED + break + case "+": + parser.currentClause.presence = lunr.Query.presence.REQUIRED + break + default: + var errorMessage = "unrecognised presence operator'" + lexeme.str + "'" + throw new lunr.QueryParseError (errorMessage, lexeme.start, lexeme.end) + } + + var nextLexeme = parser.peekLexeme() + + if (nextLexeme == undefined) { + var errorMessage = "expecting term or field, found nothing" + throw new lunr.QueryParseError (errorMessage, lexeme.start, lexeme.end) + } + + switch (nextLexeme.type) { + case lunr.QueryLexer.FIELD: + return lunr.QueryParser.parseField + case lunr.QueryLexer.TERM: + return lunr.QueryParser.parseTerm + default: + var errorMessage = "expecting term or field, found '" + nextLexeme.type + "'" + throw new lunr.QueryParseError (errorMessage, nextLexeme.start, nextLexeme.end) + } +} + +lunr.QueryParser.parseField = function (parser) { + var lexeme = parser.consumeLexeme() + + if (lexeme == undefined) { + return + } + + if (parser.query.allFields.indexOf(lexeme.str) == -1) { + var possibleFields = parser.query.allFields.map(function (f) { return "'" + f + "'" }).join(', '), + errorMessage = "unrecognised field '" + lexeme.str + "', possible fields: " + possibleFields + + throw new lunr.QueryParseError (errorMessage, lexeme.start, lexeme.end) + } + + parser.currentClause.fields = [lexeme.str] + + var nextLexeme = parser.peekLexeme() + + if (nextLexeme == undefined) { + var errorMessage = "expecting term, found nothing" + throw new lunr.QueryParseError (errorMessage, lexeme.start, lexeme.end) + } + + switch (nextLexeme.type) { + case lunr.QueryLexer.TERM: + return lunr.QueryParser.parseTerm + default: + var errorMessage = "expecting term, found '" + nextLexeme.type + "'" + throw new lunr.QueryParseError (errorMessage, nextLexeme.start, nextLexeme.end) + } +} + +lunr.QueryParser.parseTerm = function (parser) { + var lexeme = parser.consumeLexeme() + + if (lexeme == undefined) { + return + } + + parser.currentClause.term = lexeme.str.toLowerCase() + + if (lexeme.str.indexOf("*") != -1) { + parser.currentClause.usePipeline = false + } + + var nextLexeme = parser.peekLexeme() + + if (nextLexeme == undefined) { + parser.nextClause() + return + } + + switch (nextLexeme.type) { + case lunr.QueryLexer.TERM: + parser.nextClause() + return lunr.QueryParser.parseTerm + case lunr.QueryLexer.FIELD: + parser.nextClause() + return lunr.QueryParser.parseField + case lunr.QueryLexer.EDIT_DISTANCE: + return lunr.QueryParser.parseEditDistance + case lunr.QueryLexer.BOOST: + return lunr.QueryParser.parseBoost + case lunr.QueryLexer.PRESENCE: + parser.nextClause() + return lunr.QueryParser.parsePresence + default: + var errorMessage = "Unexpected lexeme type '" + nextLexeme.type + "'" + throw new lunr.QueryParseError (errorMessage, nextLexeme.start, nextLexeme.end) + } +} + +lunr.QueryParser.parseEditDistance = function (parser) { + var lexeme = parser.consumeLexeme() + + if (lexeme == undefined) { + return + } + + var editDistance = parseInt(lexeme.str, 10) + + if (isNaN(editDistance)) { + var errorMessage = "edit distance must be numeric" + throw new lunr.QueryParseError (errorMessage, lexeme.start, lexeme.end) + } + + parser.currentClause.editDistance = editDistance + + var nextLexeme = parser.peekLexeme() + + if (nextLexeme == undefined) { + parser.nextClause() + return + } + + switch (nextLexeme.type) { + case lunr.QueryLexer.TERM: + parser.nextClause() + return lunr.QueryParser.parseTerm + case lunr.QueryLexer.FIELD: + parser.nextClause() + return lunr.QueryParser.parseField + case lunr.QueryLexer.EDIT_DISTANCE: + return lunr.QueryParser.parseEditDistance + case lunr.QueryLexer.BOOST: + return lunr.QueryParser.parseBoost + case lunr.QueryLexer.PRESENCE: + parser.nextClause() + return lunr.QueryParser.parsePresence + default: + var errorMessage = "Unexpected lexeme type '" + nextLexeme.type + "'" + throw new lunr.QueryParseError (errorMessage, nextLexeme.start, nextLexeme.end) + } +} + +lunr.QueryParser.parseBoost = function (parser) { + var lexeme = parser.consumeLexeme() + + if (lexeme == undefined) { + return + } + + var boost = parseInt(lexeme.str, 10) + + if (isNaN(boost)) { + var errorMessage = "boost must be numeric" + throw new lunr.QueryParseError (errorMessage, lexeme.start, lexeme.end) + } + + parser.currentClause.boost = boost + + var nextLexeme = parser.peekLexeme() + + if (nextLexeme == undefined) { + parser.nextClause() + return + } + + switch (nextLexeme.type) { + case lunr.QueryLexer.TERM: + parser.nextClause() + return lunr.QueryParser.parseTerm + case lunr.QueryLexer.FIELD: + parser.nextClause() + return lunr.QueryParser.parseField + case lunr.QueryLexer.EDIT_DISTANCE: + return lunr.QueryParser.parseEditDistance + case lunr.QueryLexer.BOOST: + return lunr.QueryParser.parseBoost + case lunr.QueryLexer.PRESENCE: + parser.nextClause() + return lunr.QueryParser.parsePresence + default: + var errorMessage = "Unexpected lexeme type '" + nextLexeme.type + "'" + throw new lunr.QueryParseError (errorMessage, nextLexeme.start, nextLexeme.end) + } +} + + /** + * export the module via AMD, CommonJS or as a browser global + * Export code from https://github.com/umdjs/umd/blob/master/returnExports.js + */ + ;(function (root, factory) { + if (typeof define === 'function' && define.amd) { + // AMD. Register as an anonymous module. + define(factory) + } else if (typeof exports === 'object') { + /** + * Node. Does not work with strict CommonJS, but + * only CommonJS-like enviroments that support module.exports, + * like Node. + */ + module.exports = factory() + } else { + // Browser globals (root is window) + root.lunr = factory() + } + }(this, function () { + /** + * Just return a value to define the module export. + * This example returns an object, but the module + * can return a function as the exported value. + */ + return lunr + })) +})(); diff --git a/search/main.js b/search/main.js new file mode 100644 index 0000000..a5e469d --- /dev/null +++ b/search/main.js @@ -0,0 +1,109 @@ +function getSearchTermFromLocation() { + var sPageURL = window.location.search.substring(1); + var sURLVariables = sPageURL.split('&'); + for (var i = 0; i < sURLVariables.length; i++) { + var sParameterName = sURLVariables[i].split('='); + if (sParameterName[0] == 'q') { + return decodeURIComponent(sParameterName[1].replace(/\+/g, '%20')); + } + } +} + +function joinUrl (base, path) { + if (path.substring(0, 1) === "/") { + // path starts with `/`. Thus it is absolute. + return path; + } + if (base.substring(base.length-1) === "/") { + // base ends with `/` + return base + path; + } + return base + "/" + path; +} + +function escapeHtml (value) { + return value.replace(/&/g, '&') + .replace(/"/g, '"') + .replace(/</g, '<') + .replace(/>/g, '>'); +} + +function formatResult (location, title, summary) { + return '<article><h3><a href="' + joinUrl(base_url, location) + '">'+ escapeHtml(title) + '</a></h3><p>' + escapeHtml(summary) +'</p></article>'; +} + +function displayResults (results) { + var search_results = document.getElementById("mkdocs-search-results"); + while (search_results.firstChild) { + search_results.removeChild(search_results.firstChild); + } + if (results.length > 0){ + for (var i=0; i < results.length; i++){ + var result = results[i]; + var html = formatResult(result.location, result.title, result.summary); + search_results.insertAdjacentHTML('beforeend', html); + } + } else { + var noResultsText = search_results.getAttribute('data-no-results-text'); + if (!noResultsText) { + noResultsText = "No results found"; + } + search_results.insertAdjacentHTML('beforeend', '<p>' + noResultsText + '</p>'); + } +} + +function doSearch () { + var query = document.getElementById('mkdocs-search-query').value; + if (query.length > min_search_length) { + if (!window.Worker) { + displayResults(search(query)); + } else { + searchWorker.postMessage({query: query}); + } + } else { + // Clear results for short queries + displayResults([]); + } +} + +function initSearch () { + var search_input = document.getElementById('mkdocs-search-query'); + if (search_input) { + search_input.addEventListener("keyup", doSearch); + } + var term = getSearchTermFromLocation(); + if (term) { + search_input.value = term; + doSearch(); + } +} + +function onWorkerMessage (e) { + if (e.data.allowSearch) { + initSearch(); + } else if (e.data.results) { + var results = e.data.results; + displayResults(results); + } else if (e.data.config) { + min_search_length = e.data.config.min_search_length-1; + } +} + +if (!window.Worker) { + console.log('Web Worker API not supported'); + // load index in main thread + $.getScript(joinUrl(base_url, "search/worker.js")).done(function () { + console.log('Loaded worker'); + init(); + window.postMessage = function (msg) { + onWorkerMessage({data: msg}); + }; + }).fail(function (jqxhr, settings, exception) { + console.error('Could not load worker.js'); + }); +} else { + // Wrap search in a web worker + var searchWorker = new Worker(joinUrl(base_url, "search/worker.js")); + searchWorker.postMessage({init: true}); + searchWorker.onmessage = onWorkerMessage; +} diff --git a/search/search_index.json b/search/search_index.json new file mode 100644 index 0000000..065d4fc --- /dev/null +++ b/search/search_index.json @@ -0,0 +1 @@ +{"config":{"indexing":"full","lang":["en"],"min_search_length":3,"prebuild_index":false,"separator":"[\\s\\-]+"},"docs":[{"location":"index.html","text":"About # This chapter contains background information for the library. We give a general introduction to eduVPN and explain what problems this library aims to solve. EduVPN introduction # eduVPN-common is a library for eduVPN , which is a VPN by SURF and a project by G\u00c9ANT , for research institutes and universities. Each institute that uses eduVPN has its own server. To discover these servers and establish a VPN connection with them, eduVPN clients are used. eduVPN has clients for each common platform: - Android - Linux - MacOS/iOS - Windows The problem # However, as these clients are rather similar in functionality, apart from platform specific differences, right now there is duplicate code between them. For example, the process to discover institution\u2019s servers, the authorization process (OAuth) and Wireguard key generation. This goal of this library is to provide the common functionality between these clients into one codebase. The library is written in the Go language and aims to have wrapper code for each of the languages that are used by the current clients. The main goal is thus the following: This library tries to remove non-platform specific common functionality. This way eduVPN clients have less duplicate code. The building blocks that are removed by the library is not just the four depicted in this figure. You can think of other building blocks, such as logging and local configuration file saving. As can be seen in the figure, no User Interface (UI) code will be implemented. This is left to the eduVPN clients, on top of platform-specific code. License # MIT Authors # This library is written by Steven Wallis de Vries and Jeroen Wijenbergh at the SURF and G\u00c9ANT organization.","title":"About"},{"location":"index.html#eduvpn-introduction","text":"eduVPN-common is a library for eduVPN , which is a VPN by SURF and a project by G\u00c9ANT , for research institutes and universities. Each institute that uses eduVPN has its own server. To discover these servers and establish a VPN connection with them, eduVPN clients are used. eduVPN has clients for each common platform: - Android - Linux - MacOS/iOS - Windows","title":"EduVPN introduction"},{"location":"index.html#the-problem","text":"However, as these clients are rather similar in functionality, apart from platform specific differences, right now there is duplicate code between them. For example, the process to discover institution\u2019s servers, the authorization process (OAuth) and Wireguard key generation. This goal of this library is to provide the common functionality between these clients into one codebase. The library is written in the Go language and aims to have wrapper code for each of the languages that are used by the current clients. The main goal is thus the following: This library tries to remove non-platform specific common functionality. This way eduVPN clients have less duplicate code. The building blocks that are removed by the library is not just the four depicted in this figure. You can think of other building blocks, such as logging and local configuration file saving. As can be seen in the figure, no User Interface (UI) code will be implemented. This is left to the eduVPN clients, on top of platform-specific code.","title":"The problem"},{"location":"index.html#license","text":"MIT","title":"License"},{"location":"index.html#authors","text":"This library is written by Steven Wallis de Vries and Jeroen Wijenbergh at the SURF and G\u00c9ANT organization.","title":"Authors"},{"location":"apidocs.html","text":"This document was automatically generated from the exports/exports.go file About the API # Some notes: Errors are returned as JSON c strings. The JSON type is defined in types/error/error.go Error . Free them using FreeString . Same is the case for other string types, you should also free them. The errors are always localized Types are converted from the Go representation to C using JSON strings Cookies are used for cancellation, just fancy contexts. Create a cookie using CookieNew , pass it to the function that needs one as the first argument. To cancel the function, call CookieCancel , passing in the same cookie as argument Cookies must also be freed, by using the CookieDelete function if the cookie is no longer needed The state machine is used to track the state of a client. It is mainly used for asking for certain data from the client, e.g. asking for profiles and locations. But a client may also wish to build upon this state machine to build the whole UI around it. The SetState and InState functions are useful for this Functions # AddServer # Signature: func AddServer(c C.uintptr_t, _type C.int, id *C.char, ot *C.longlong) *C.char AddServer adds a server to the eduvpn-common server list c is the cookie that is used for cancellation. Create a cookie first with CookieNew. This same cookie is also used for replying to state transitions. _type is the type of server that needs to be added. This type is defined in types/server/server.go Type id is the identifier of the string: In case of secure internet: The organization ID In case of custom server: The base URL In case of institute access: The base URL ni stands for non-interactive. If non-zero, any state transitions will not be run. This ot flag is useful for preprovisioned servers; set this to non-null to non-interactively add a server. This flag represents the Unix time OAuth was last triggered, if the server needs to be added non-interactively but there is no token structure, set this to zero (integer) or the current Unix time. This value will be overwritten once OAuth is triggered. If the server cannot be added it returns the error as types/error/error.go Error . Note that the server is removed when an error has occured The following state callbacks are mandatory to handle: OAUTH_STARTED: This indicates that the OAuth procedure has been started, it returns the URL as the data. The client should open the webbrowser with this URL and continue the authorization process. Note: For mobile platforms this returns a Cookie and data (json: {\"cookie\": x, \"data\": url} ). This url should also be opened in the browser like desktop platforms. But these platforms also need to reply to the library to give back the full authorization code URI with CookieReply(x, uri) . E.g. CookieReply(x, \"/callback?code=...&state=...&iss=...\") this is the path of the request that the apps get back when the user clicks approve. For this, apps need to register an app url or sorts. For the valid values for app URLs, see the redirect URIs for mobile platforms here https://git.sr.ht/~fkooman/vpn-user-portal/tree/v3/item/src/OAuth/VpnClientDb.php Example Input (3=custom server): AddServer(mycookie, 3, \"https://demo.eduvpn.nl\", 0) Example Output: { \"message\": { \"en\": \"failed to add server\" }, \"misc\": false } CalculateGateway # Signature: func CalculateGateway(subnet *C.char) (*C.char, *C.char) CalculateGateway calculates the gateway for a subnet, it can take IPv4 or IPv6 networks with CIDR notation as inputs and returns the gateway address. This is useful to pass to StartFailover . It returns an error if it fails to calculate a gateway. The function is implemented according to: the eduVPN docs . Example Input: CalculateGateway(\"10.10.0.5/24\") Example Output: \"10.10.0.1\", null Cleanup # Signature: func Cleanup(c C.uintptr_t) *C.char Cleanup sends a /disconnect to cleanup the connection. This MUST be called when disconnecting, see the eduVPN docs . c is the Cookie that needs to be passed. Create a new Cookie using CookieNew . If it was unsuccessful, it returns an error. Example Input: Cleanup(myCookie) Example Output: { \"message\": { \"en\": \"cleanup was not successful\" }, \"misc\": false } CookieCancel # Signature: func CookieCancel(c C.uintptr_t) *C.char CookieCancel cancels the cookie. This means that functions which take this as first argument, return if they\u2019re still running. The error cause is always context.Canceled for that cancelled function: see the Go docs . This CookieCancel function can also return an error if cancelling was unsuccessful. Example Input: CookieCancel(myCookie) Example Output: null CookieDelete # Signature: func CookieDelete(c C.uintptr_t) *C.char CookieDelete deletes the cookie by cancelling it and deleting the underlying cgo handle. This function MUST be called when the cookie that is created using CookieNew is no longer needed. Example Input: CookieDelete(myCookie) Example Output: null CookieNew # Signature: func CookieNew() C.uintptr_t CookieNew creates a new cookie and returns it. Functions that take a cookie have it as the first argument. This value should not be parsed or converted somehow by the client. This value is simply to pass back to the Go library. This value has two purposes: Cancel a long running function Send a reply to a state transition (ASK_PROFILE and ASK_LOCATION) Example Input: CookieNew() Example Output: 5 CookieReply # Signature: func CookieReply(c C.uintptr_t, data *C.char) *C.char CookieReply replies to a state transition using the cookie. c is the Cookie data is the data to send, e.g. a profile ID Example Input: CookieReply(myCookie, \"split-tunnel-profile\") Example Output: null CurrentServer # Signature: func CurrentServer() (*C.char, *C.char) CurrentServer gets the current server from eduvpn-common In eduvpn-common, a server is marked as \u2018current\u2019 if you have gotten a VPN configuration for it It returns the server as JSON, defined in types/server/server.go Current . If there is no current server or some other, e.g. there is no current state, an error is returned with a nil string. Example Input: CurrentServer() Example Output: { \"institute_access_server\": { \"display_name\": { \"en\": \"Demo\" }, \"identifier\": \"https://demo.eduvpn.nl/\", \"profiles\": { \"map\": { \"internet\": { \"display_name\": { \"en\": \"Internet\" }, \"supported_protocols\": [ 1, 2 ] }, \"internet-split\": { \"display_name\": { \"en\": \"No rfc1918 routes\" }, \"supported_protocols\": [ 1, 2 ] } }, \"current\": \"internet\" }, \"support_contacts\": [ \"mailto:eduvpn@surf.nl\" ], \"delisted\": false }, \"server_type\": 1 }, null Deregister # Signature: func Deregister() *C.char Deregister cleans up the state for the client. This function SHOULD be called when the application exits such that the configuration file is saved correctly. Note that saving of the configuration file also happens in other cases, such as after getting a VPN configuration. Thus it is often not problematic if this function cannot be called due to a client crash. If no client is available or deregistering fails, it returns an error. Example Input: Deregister() Example Output: { \"message\": { \"en\": \"failed to deregister\" }, \"misc\": false } DiscoOrganizations # Signature: func DiscoOrganizations(c C.uintptr_t, cache C.int, search *C.char) (*C.char, *C.char) DiscoOrganizations gets the organizations from discovery, returned as types/discovery/discovery.go Organizations marshalled as JSON. c is the Cookie that needs to be passed. Create a new Cookie using CookieNew cache indicates whether or not the cache should only be used, meaning no network call search is the search string for filtering the list. If any of the words in the search query is not contained in any of the display names or keywords, the candidate is filtered. Otherwise they are ranked based on the levenshtein distance: Levenshtein Wikipedia . If search is empty it returns ALL organizations currently known in common If it was unsuccessful, it returns an error. Note that when the lib was built in release mode the data is almost always non-nil, even when an error has occurred This means it has just returned the cached list, the error should then not be handled in a fatal way. E.g. show the returned cache list but log the error or show the error with a warning. Example Input: DiscoOrganizations(myCookie, \"\") Example Output: { \"organization_list\": [ { \"display_name\": { \"en\": \"Academic Network of Albania - RASH\" }, \"org_id\": \"https://idp.rash.al/simplesaml/saml2/idp/metadata.php\", }, { \"display_name\": { \"da\": \"Dansk Sprogn\u00e6vn\", \"en\": \"Danish Language Council\" }, \"org_id\": \"http://idp.dsn.dk/adfs/services/trust\", }, { \"display_name\": { \"da\": \"Erhvervsakademi Aarhus\", \"en\": \"Business Academy Aarhus\" }, \"org_id\": \"http://adfs.eaaa.dk/adfs/services/trust\", }, null Example Input: DiscoOrganizations(myCookie, \"rash\") Example Output: { \"organization_list\": [ { \"display_name\": { \"en\": \"Academic Network of Albania - RASH\" }, \"org_id\": \"https://idp.rash.al/simplesaml/saml2/idp/metadata.php\", }, ] }, null DiscoServers # Signature: func DiscoServers(c C.uintptr_t, cache C.int, search *C.char) (*C.char, *C.char) DiscoServers gets the servers from discovery, returned as types/discovery/discovery.go Servers marshalled as JSON c is the Cookie that needs to be passed. Create a new Cookie using CookieNew cache indicates whether or not the cache should only be used, meaning no network call search is the search string for filtering the list. If any of the words in the search query is not contained in any of the display names or keywords, the candidate is filtered. Otherwise they are ranked based on the levenshtein distance: Levenshtein Wikipedia . If search is empty it returns ALL servers currently known in common If it was unsuccessful, it returns an error. Note that when the lib was built in release mode the data is almost always non-nil, even when an error has occurred. This means it has just returned the cached list, the error should then not be handled in a fatal way. E.g. show the returned cache list but log the error or show the error with a warning. Example Input: DiscoServers(myCookie, \"\") Example Output: { \"server_list\": [ { \"base_url\": \"https://eduvpn.rash.al/\", \"country_code\": \"AL\", \"server_type\": \"secure_internet\", }, { \"base_url\": \"https://eduvpn.deic.dk/\", \"country_code\": \"DK\", \"server_type\": \"secure_internet\", } , null Example Input: DiscoServers(myCookie, \"heanet\") Example Output: { \"server_list\": [ { \"base_url\": \"https://eduvpn.heanet.ie/\", \"display_name\": { \"en\": \"HEAnet Staff\" }, \"server_type\": \"institute_access\", }, ] } , null ExpiryTimes # Signature: func ExpiryTimes() (*C.char, *C.char) ExpiryTimes gets the expiry times for the current server Expiry times are just fields that represent unix timestamps at which to do certain events regarding expiry, e.g. when to show the renew button, when to show expiry notifications The expiry times structure is defined in types/server/server.go Expiry If some error occurs, it is returned as types/error/error.go Error Example Input: ExpiryTimes() Example Output (1\u20264 are unix timestamps): { \"start_time\": 1, \"end_time\": 2, \"button_time\": 3, \"countdown_time\": 4, \"notification_times\": [ 1, 2, ], }, null FreeString # Signature: func FreeString(addr *C.char) FreeString frees a string that was allocated by the eduvpn-common Go library. This happens when we return strings, such as errors from the Go lib back to the client. The client MUST thus ensure that this memory is freed using this function. Simply pass the pointer to the string in here. Example Input: FreeString(strPtr) GetConfig # Signature: func GetConfig(c C.uintptr_t, _type C.int, id *C.char, pTCP C.int, startup C.int) (*C.char, *C.char) GetConfig gets a configuration for the server. c is the cookie that is used for cancellation. Create a cookie first with CookieNew, this same cookie is also used for replying to state transitions _type is the type of server that needs to be added. This type is defined in types/server/server.go Type id is the identifier of the string In case of secure internet: The organization ID In case of custom server: The base URL In case of institute access: The base URL pTCP is if we prefer TCP or not to get the configuration, non-zero means yes startup is if the client is just starting up, set this to true (non-zero) if you autoconnect to a server on startup. If this startup value is true (non-zero) then any authorization or other callacks (profile/location) are not triggered After getting a configuration, the FSM moves to the GOT_CONFIG state The return data is the configuration, marshalled as JSON and defined in types/server/server.go Configuration If the config cannot be retrieved it returns an error as types/error/error.go Error . The current state callbacks MUST be handled: ASK_PROFILE # This asks the client for profile. This is called when the user/client has not set a profile for this server before, or the current profile is invalid When the user has selected a profile, reply with the choice using the CookieReply function and the profile ID e.g. CookieReply(cookie, \u201cwireguard\u201d). CookieReply can be done in the background as the Go library waits for a reply The data for this transition is defined in types/server/server.go RequiredAskTransition with embedded data Profiles in types/server/server.go . Note that RequiredAskTransition contains the cookie to be used for the CookieReply . So a client would: Parse the data to get the cookie and data get the cookie get the profiles from the data show it in the UI and then reply with CookieReply using the choice ASK_LOCATION # This asks the client for a location. Note that under normal circumstances, this callback is not actually called as the home organization for the secure internet server is set as the current if for some reason, an invalid location has been configured, the library will ask the client for a new one When the user has selected a location, reply with the choice using the CookieReply function and the location ID e.g. CookieReply(cookie, \u201cnl\u201d) CookieReply can be done in the background as the Go library waits for a reply The data for this transition is defined in types/server/server.go RequiredAskTransition with embedded data a list of strings ( []string ) Note that RequiredAskTransition contains the cookie to be used for the CookieReply function, So a client would: Parse the data to get the cookie and data get the cookie get the list of locations from the data show it in the UI and then reply with CookieReply using the choice OAUTH_STARTED # OAUTH_STARTED: This indicates that the OAuth procedure has been started, it returns the URL as the data. The client should open the webbrowser with this URL and continue the authorization process. Note: For mobile platforms this returns a Cookie and data (json: {\"cookie\": x, \"data\": url} ). This url should also be opened in the browser like desktop platforms. But these platforms also need to reply to the library to give back the full authorization code URI with CookieReply(x, uri) . E.g. CookieReply(x, \"/callback?code=...&state=...&iss=...\") this is the path of the request that the apps get back when the user clicks approve. For this, apps need to register an app url or sorts. For the valid values for app URLs, see the redirect URIs for mobile platforms here https://git.sr.ht/~fkooman/vpn-user-portal/tree/v3/item/src/OAuth/VpnClientDb.php The client should open the webbrowser with this URL and continue the authorization process. This is only called if authorization needs to be retriggered Example Input (3=custom server): GetConfig(myCookie, 3, \"https://demo.eduvpn.nl/\", 0, 0) Example Output (2=WireGuard): { \"config\": \"[Interface]\\nPrivateKey = ...\\nAddress = ...\\nDNS = ...\\n\\n[Peer]\\nPublicKey = ...=\\nAllowedIPs = 0.0.0.0/0,::/0\\nEndpoint = ...\", \"protocol\": 2, \"default_gateway\": true, \"should_failover\": true, <- whether or not the failover procedure should happen } InState # Signature: func InState(fsmState C.int) (C.int, *C.char) InState checks if the FSM is in fsmState . Example Input: InState(5) Example Output: 1, null NewProxyguard # Signature: func NewProxyguard(c C.uintptr_t, lp C.int, tcpsp C.int, peer *C.char, proxySetup C.ProxySetup) (C.uintptr_t, *C.char) NewProxyguard creates the \u2018proxyguard\u2019 procedure in eduvpn-common. If the proxy cannot be created it returns an error. This function proxies WireGuard UDP connections over HTTP: ProxyGuard on Codeberg . These input variables can be gotten from the configuration that is retrieved using the proxy JSON key c is the cookie. Note that if you cancel/delete the cookie, ProxyGuard gets cleaned up. Common automatically cleans up ProxyGuard when Cleanup is called, but it is good to cleanup yourself too. lp is the port of the local udp ProxyGuard connection, this is what is set to the WireGuard endpoint tcpsp is the TCP source port. Pass 0 if you do not route based on source port, so far only the Linux client has to pass non-zero. peer is the ip:port of the remote server proxySetup is a callback which is called when the socket is setting up, this can be used for configuring routing in the client. It takes two arguments: the file descriptor (integer) and a JSON list of IPs the client connects to Example Input: NewProxyguard(myCookie, 1337, 0, \"5.5.5.5:51820\", proxySetupCB) Example Output: null ProxyguardPeerIPs # Signature: func ProxyguardPeerIPs(proxyH C.uintptr_t) (*C.char, *C.char) ProxyguardPeerIPs gets the Peer IPs configured by ProxyGuard Example Input: ProxyguardPeerIPs(handle) Example Output: [\"1.1.1.1\"], null ProxyguardRestart # Signature: func ProxyguardRestart(proxyH C.uintptr_t) *C.char ProxyguardRestart restarts ProxyGuard, call this when a network change happens Example Input: ProxyguardRestart(proxyHandle) Example Output: \"failed restarting ProxyGuard\" ProxyguardTunnel # Signature: func ProxyguardTunnel(c C.uintptr_t, proxyH C.uintptr_t, wglisten C.int) *C.char ProxyguardTunnel starts the tunneling for ProxyGuard c is the cookie proxyH is the proxy handle wglisten is the port WireGuard is listening on Register # Signature: func Register( name *C.char, version *C.char, configDirectory *C.char, cb C.StateCB, ) *C.char Register creates a new client and also registers the FSM to go to the initial state Name is the name of the client, must be a valid client ID. Version is the version of the client. This version field is used for the user agent in all HTTP requests. cb is the state callback. It takes three arguments: The old state, the new state and the data for the state as JSON. Note that the states are defined in client/fsm.go, e.g. Main (in Go: StateMain ), ASK_PROFILE (in Go: StateAskProfile ) This callback returns non-zero if the state transition is handled. This is used to check if the client handles the needed transitions After registering, the FSM is initialized and the state transition MAIN should have been completed If some error occurs during registering, it is returned as a types/error/error.go Error Example Input: Register(\"org.eduvpn.app.linux\", \"0.0.1\", \"/tmp/eduvpn-common\", myCallbackFunc, 1) Example Output: { \"message\": { \"en\": \"failed to register, a VPN state is already present\" }, \"misc\": false } RemoveServer # Signature: func RemoveServer(_type C.int, id *C.char) *C.char RemoveServer removes a server from the eduvpn-common server list _type is the type of server that needs to be added. This type is defined in types/server/server.go Type id is the identifier of the string: In case of secure internet: The organization ID In case of custom server: The base URL In case of institute access: The base URL If the server cannot be removed it returns the error types/error/error.go Error . Example Input (3=custom server): RemoveServer(3, \"bogus\") Example Output: { \"message\": { \"en\": \"failed to remove server\" }, \"misc\": false } RenewSession # Signature: func RenewSession(c C.uintptr_t) *C.char RenewSession renews the session of the VPN This essentially means that the OAuth tokens are deleted. And it also possibly re-runs every state callback you need when getting a config. So least you MUST handle the OAuth started transition It returns an error if unsuccessful. Example Input: RenewSession(myCookie) Example Output: { \"message\": { \"en\": \"could not renew session\" }, \"misc\": false } ServerList # Signature: func ServerList() (*C.char, *C.char) ServerList gets the list of servers that are currently added This is NOT the discovery list, but the servers that have previously been added with AddServer . It returns the server list as a JSON string defined in types/server/server.go List . If the server list cannot be retrieved it returns a nil string and an error. Example Input: ServerList() Example Output (current profile here is empty as none has been chosen yet): { \"institute_access_servers\": [ { \"display_name\": { \"en\": \"Demo\" }, \"identifier\": \"https://demo.eduvpn.nl/\", \"profiles\": { \"current\": \"\" }, \"support_contacts\": [ \"mailto:eduvpn@surf.nl\" ], \"delisted\": false } ] }, null SetProfileID # Signature: func SetProfileID(data *C.char) *C.char SetProfileID sets the profile ID of the current serrver. This MUST only be called if the user/client wishes to manually set a profile instead of the common lib asking for one using a transition. data is the profile ID. It returns an error if unsuccessful. Example Input: SetProfileID(\"splittunnel\") Example Output: { \"message\": { \"en\": \"profile does not exist\" }, \"misc\": false } SetSecureLocation # Signature: func SetSecureLocation(orgID *C.char, cc *C.char) *C.char SetSecureLocation sets the location for the secure internet server if it exists. This MUST only be called if the user/client wishes to manually set a location instead of the common lib asking for one using a transition. orgID is the organisation ID for the secure internet server cc is the location ID/country code It returns an error if unsuccessful. Example Input: SetSecureLocation(\"http://idp.geant.org/\", \"nl\") Example Output: { \"message\": { \"en\": \"location does not exist\" }, \"misc\": false } SetState # Signature: func SetState(fsmState C.int) *C.char SetState sets the state of the state machine. Note: this transitions the FSM into the new state without passing any data to it. Example Input: SetState(5) Example Output: null SetTokenHandler # Signature: func SetTokenHandler(getter C.TokenGetter, setter C.TokenSetter) *C.char SetTokenHandler sets the token getters and token setters for OAuth. Because the data that is saved does not contain OAuth tokens for server, the common lib asks and sets the tokens using these callback functions. The client can thus pass callbacks to this function so that the tokens can be securely stored in a keyring. The client must pass two callback arguments to this function: getter is the void function that gets tokens from the client. It takes three arguments: - The server for which to get the tokens for, marshalled as JSON and defined in types/server/server.go Current - The output buffer - The length of the output buffer. This \u2018output buffer\u2019 must contain the tokens, marshalled as JSON that is defined in types/server/server.go Tokens setter is the void function that sets tokens. It takes two arguments: The server for which to get the tokens for, marshalled as JSON and defined in types/server/server.go Current The tokens , defined in types/server/server.go Tokens marshalled as JSON It returns an error when the tokens cannot be set. Example Input: SetTokenHandler(getterFunc, setterFunc) Example Output: null StartFailover # Signature: func StartFailover(c C.uintptr_t, gateway *C.char, mtu C.int, readRxBytes C.ReadRxBytes) (C.int, *C.char) StartFailover starts the \u2018failover\u2019 procedure in eduvpn-common. Failover has one primary goal: check if the VPN can reach the gateway. This can be used to check whether or not the client needs to \u2018failover\u2019 to prefer TCP (if currently using UDP). Which is useful to go from a broken WireGuard connection to OpenVPN over TCP. c is the cookie that is passed for cancellation. To create a cookie, use the CookieNew function gateway is the gateway IP of the VPN. You MAY calculate this with the CalculateGateway function readRxBytes is a function that returns the current rx bytes of the VPN interface, this should return a long long int in c It returns a boolean whether or not the common lib has determined that it cannot reach the gateway. Non-zero=dropped, zero=not dropped. It also returns an error, if it fails to indicate if it has dropped or not. In this case, dropped is also set to zero. Example Input: StartFailover(myCookie, \"10.10.10.1\", 1400, myRxBytesReader) Example Output: 1, null","title":"API Docs"},{"location":"apidocs.html#addserver","text":"Signature: func AddServer(c C.uintptr_t, _type C.int, id *C.char, ot *C.longlong) *C.char AddServer adds a server to the eduvpn-common server list c is the cookie that is used for cancellation. Create a cookie first with CookieNew. This same cookie is also used for replying to state transitions. _type is the type of server that needs to be added. This type is defined in types/server/server.go Type id is the identifier of the string: In case of secure internet: The organization ID In case of custom server: The base URL In case of institute access: The base URL ni stands for non-interactive. If non-zero, any state transitions will not be run. This ot flag is useful for preprovisioned servers; set this to non-null to non-interactively add a server. This flag represents the Unix time OAuth was last triggered, if the server needs to be added non-interactively but there is no token structure, set this to zero (integer) or the current Unix time. This value will be overwritten once OAuth is triggered. If the server cannot be added it returns the error as types/error/error.go Error . Note that the server is removed when an error has occured The following state callbacks are mandatory to handle: OAUTH_STARTED: This indicates that the OAuth procedure has been started, it returns the URL as the data. The client should open the webbrowser with this URL and continue the authorization process. Note: For mobile platforms this returns a Cookie and data (json: {\"cookie\": x, \"data\": url} ). This url should also be opened in the browser like desktop platforms. But these platforms also need to reply to the library to give back the full authorization code URI with CookieReply(x, uri) . E.g. CookieReply(x, \"/callback?code=...&state=...&iss=...\") this is the path of the request that the apps get back when the user clicks approve. For this, apps need to register an app url or sorts. For the valid values for app URLs, see the redirect URIs for mobile platforms here https://git.sr.ht/~fkooman/vpn-user-portal/tree/v3/item/src/OAuth/VpnClientDb.php Example Input (3=custom server): AddServer(mycookie, 3, \"https://demo.eduvpn.nl\", 0) Example Output: { \"message\": { \"en\": \"failed to add server\" }, \"misc\": false }","title":"AddServer"},{"location":"apidocs.html#calculategateway","text":"Signature: func CalculateGateway(subnet *C.char) (*C.char, *C.char) CalculateGateway calculates the gateway for a subnet, it can take IPv4 or IPv6 networks with CIDR notation as inputs and returns the gateway address. This is useful to pass to StartFailover . It returns an error if it fails to calculate a gateway. The function is implemented according to: the eduVPN docs . Example Input: CalculateGateway(\"10.10.0.5/24\") Example Output: \"10.10.0.1\", null","title":"CalculateGateway"},{"location":"apidocs.html#cleanup","text":"Signature: func Cleanup(c C.uintptr_t) *C.char Cleanup sends a /disconnect to cleanup the connection. This MUST be called when disconnecting, see the eduVPN docs . c is the Cookie that needs to be passed. Create a new Cookie using CookieNew . If it was unsuccessful, it returns an error. Example Input: Cleanup(myCookie) Example Output: { \"message\": { \"en\": \"cleanup was not successful\" }, \"misc\": false }","title":"Cleanup"},{"location":"apidocs.html#cookiecancel","text":"Signature: func CookieCancel(c C.uintptr_t) *C.char CookieCancel cancels the cookie. This means that functions which take this as first argument, return if they\u2019re still running. The error cause is always context.Canceled for that cancelled function: see the Go docs . This CookieCancel function can also return an error if cancelling was unsuccessful. Example Input: CookieCancel(myCookie) Example Output: null","title":"CookieCancel"},{"location":"apidocs.html#cookiedelete","text":"Signature: func CookieDelete(c C.uintptr_t) *C.char CookieDelete deletes the cookie by cancelling it and deleting the underlying cgo handle. This function MUST be called when the cookie that is created using CookieNew is no longer needed. Example Input: CookieDelete(myCookie) Example Output: null","title":"CookieDelete"},{"location":"apidocs.html#cookienew","text":"Signature: func CookieNew() C.uintptr_t CookieNew creates a new cookie and returns it. Functions that take a cookie have it as the first argument. This value should not be parsed or converted somehow by the client. This value is simply to pass back to the Go library. This value has two purposes: Cancel a long running function Send a reply to a state transition (ASK_PROFILE and ASK_LOCATION) Example Input: CookieNew() Example Output: 5","title":"CookieNew"},{"location":"apidocs.html#cookiereply","text":"Signature: func CookieReply(c C.uintptr_t, data *C.char) *C.char CookieReply replies to a state transition using the cookie. c is the Cookie data is the data to send, e.g. a profile ID Example Input: CookieReply(myCookie, \"split-tunnel-profile\") Example Output: null","title":"CookieReply"},{"location":"apidocs.html#currentserver","text":"Signature: func CurrentServer() (*C.char, *C.char) CurrentServer gets the current server from eduvpn-common In eduvpn-common, a server is marked as \u2018current\u2019 if you have gotten a VPN configuration for it It returns the server as JSON, defined in types/server/server.go Current . If there is no current server or some other, e.g. there is no current state, an error is returned with a nil string. Example Input: CurrentServer() Example Output: { \"institute_access_server\": { \"display_name\": { \"en\": \"Demo\" }, \"identifier\": \"https://demo.eduvpn.nl/\", \"profiles\": { \"map\": { \"internet\": { \"display_name\": { \"en\": \"Internet\" }, \"supported_protocols\": [ 1, 2 ] }, \"internet-split\": { \"display_name\": { \"en\": \"No rfc1918 routes\" }, \"supported_protocols\": [ 1, 2 ] } }, \"current\": \"internet\" }, \"support_contacts\": [ \"mailto:eduvpn@surf.nl\" ], \"delisted\": false }, \"server_type\": 1 }, null","title":"CurrentServer"},{"location":"apidocs.html#deregister","text":"Signature: func Deregister() *C.char Deregister cleans up the state for the client. This function SHOULD be called when the application exits such that the configuration file is saved correctly. Note that saving of the configuration file also happens in other cases, such as after getting a VPN configuration. Thus it is often not problematic if this function cannot be called due to a client crash. If no client is available or deregistering fails, it returns an error. Example Input: Deregister() Example Output: { \"message\": { \"en\": \"failed to deregister\" }, \"misc\": false }","title":"Deregister"},{"location":"apidocs.html#discoorganizations","text":"Signature: func DiscoOrganizations(c C.uintptr_t, cache C.int, search *C.char) (*C.char, *C.char) DiscoOrganizations gets the organizations from discovery, returned as types/discovery/discovery.go Organizations marshalled as JSON. c is the Cookie that needs to be passed. Create a new Cookie using CookieNew cache indicates whether or not the cache should only be used, meaning no network call search is the search string for filtering the list. If any of the words in the search query is not contained in any of the display names or keywords, the candidate is filtered. Otherwise they are ranked based on the levenshtein distance: Levenshtein Wikipedia . If search is empty it returns ALL organizations currently known in common If it was unsuccessful, it returns an error. Note that when the lib was built in release mode the data is almost always non-nil, even when an error has occurred This means it has just returned the cached list, the error should then not be handled in a fatal way. E.g. show the returned cache list but log the error or show the error with a warning. Example Input: DiscoOrganizations(myCookie, \"\") Example Output: { \"organization_list\": [ { \"display_name\": { \"en\": \"Academic Network of Albania - RASH\" }, \"org_id\": \"https://idp.rash.al/simplesaml/saml2/idp/metadata.php\", }, { \"display_name\": { \"da\": \"Dansk Sprogn\u00e6vn\", \"en\": \"Danish Language Council\" }, \"org_id\": \"http://idp.dsn.dk/adfs/services/trust\", }, { \"display_name\": { \"da\": \"Erhvervsakademi Aarhus\", \"en\": \"Business Academy Aarhus\" }, \"org_id\": \"http://adfs.eaaa.dk/adfs/services/trust\", }, null Example Input: DiscoOrganizations(myCookie, \"rash\") Example Output: { \"organization_list\": [ { \"display_name\": { \"en\": \"Academic Network of Albania - RASH\" }, \"org_id\": \"https://idp.rash.al/simplesaml/saml2/idp/metadata.php\", }, ] }, null","title":"DiscoOrganizations"},{"location":"apidocs.html#discoservers","text":"Signature: func DiscoServers(c C.uintptr_t, cache C.int, search *C.char) (*C.char, *C.char) DiscoServers gets the servers from discovery, returned as types/discovery/discovery.go Servers marshalled as JSON c is the Cookie that needs to be passed. Create a new Cookie using CookieNew cache indicates whether or not the cache should only be used, meaning no network call search is the search string for filtering the list. If any of the words in the search query is not contained in any of the display names or keywords, the candidate is filtered. Otherwise they are ranked based on the levenshtein distance: Levenshtein Wikipedia . If search is empty it returns ALL servers currently known in common If it was unsuccessful, it returns an error. Note that when the lib was built in release mode the data is almost always non-nil, even when an error has occurred. This means it has just returned the cached list, the error should then not be handled in a fatal way. E.g. show the returned cache list but log the error or show the error with a warning. Example Input: DiscoServers(myCookie, \"\") Example Output: { \"server_list\": [ { \"base_url\": \"https://eduvpn.rash.al/\", \"country_code\": \"AL\", \"server_type\": \"secure_internet\", }, { \"base_url\": \"https://eduvpn.deic.dk/\", \"country_code\": \"DK\", \"server_type\": \"secure_internet\", } , null Example Input: DiscoServers(myCookie, \"heanet\") Example Output: { \"server_list\": [ { \"base_url\": \"https://eduvpn.heanet.ie/\", \"display_name\": { \"en\": \"HEAnet Staff\" }, \"server_type\": \"institute_access\", }, ] } , null","title":"DiscoServers"},{"location":"apidocs.html#expirytimes","text":"Signature: func ExpiryTimes() (*C.char, *C.char) ExpiryTimes gets the expiry times for the current server Expiry times are just fields that represent unix timestamps at which to do certain events regarding expiry, e.g. when to show the renew button, when to show expiry notifications The expiry times structure is defined in types/server/server.go Expiry If some error occurs, it is returned as types/error/error.go Error Example Input: ExpiryTimes() Example Output (1\u20264 are unix timestamps): { \"start_time\": 1, \"end_time\": 2, \"button_time\": 3, \"countdown_time\": 4, \"notification_times\": [ 1, 2, ], }, null","title":"ExpiryTimes"},{"location":"apidocs.html#freestring","text":"Signature: func FreeString(addr *C.char) FreeString frees a string that was allocated by the eduvpn-common Go library. This happens when we return strings, such as errors from the Go lib back to the client. The client MUST thus ensure that this memory is freed using this function. Simply pass the pointer to the string in here. Example Input: FreeString(strPtr)","title":"FreeString"},{"location":"apidocs.html#getconfig","text":"Signature: func GetConfig(c C.uintptr_t, _type C.int, id *C.char, pTCP C.int, startup C.int) (*C.char, *C.char) GetConfig gets a configuration for the server. c is the cookie that is used for cancellation. Create a cookie first with CookieNew, this same cookie is also used for replying to state transitions _type is the type of server that needs to be added. This type is defined in types/server/server.go Type id is the identifier of the string In case of secure internet: The organization ID In case of custom server: The base URL In case of institute access: The base URL pTCP is if we prefer TCP or not to get the configuration, non-zero means yes startup is if the client is just starting up, set this to true (non-zero) if you autoconnect to a server on startup. If this startup value is true (non-zero) then any authorization or other callacks (profile/location) are not triggered After getting a configuration, the FSM moves to the GOT_CONFIG state The return data is the configuration, marshalled as JSON and defined in types/server/server.go Configuration If the config cannot be retrieved it returns an error as types/error/error.go Error . The current state callbacks MUST be handled:","title":"GetConfig"},{"location":"apidocs.html#ask_profile","text":"This asks the client for profile. This is called when the user/client has not set a profile for this server before, or the current profile is invalid When the user has selected a profile, reply with the choice using the CookieReply function and the profile ID e.g. CookieReply(cookie, \u201cwireguard\u201d). CookieReply can be done in the background as the Go library waits for a reply The data for this transition is defined in types/server/server.go RequiredAskTransition with embedded data Profiles in types/server/server.go . Note that RequiredAskTransition contains the cookie to be used for the CookieReply . So a client would: Parse the data to get the cookie and data get the cookie get the profiles from the data show it in the UI and then reply with CookieReply using the choice","title":"ASK_PROFILE"},{"location":"apidocs.html#ask_location","text":"This asks the client for a location. Note that under normal circumstances, this callback is not actually called as the home organization for the secure internet server is set as the current if for some reason, an invalid location has been configured, the library will ask the client for a new one When the user has selected a location, reply with the choice using the CookieReply function and the location ID e.g. CookieReply(cookie, \u201cnl\u201d) CookieReply can be done in the background as the Go library waits for a reply The data for this transition is defined in types/server/server.go RequiredAskTransition with embedded data a list of strings ( []string ) Note that RequiredAskTransition contains the cookie to be used for the CookieReply function, So a client would: Parse the data to get the cookie and data get the cookie get the list of locations from the data show it in the UI and then reply with CookieReply using the choice","title":"ASK_LOCATION"},{"location":"apidocs.html#oauth_started","text":"OAUTH_STARTED: This indicates that the OAuth procedure has been started, it returns the URL as the data. The client should open the webbrowser with this URL and continue the authorization process. Note: For mobile platforms this returns a Cookie and data (json: {\"cookie\": x, \"data\": url} ). This url should also be opened in the browser like desktop platforms. But these platforms also need to reply to the library to give back the full authorization code URI with CookieReply(x, uri) . E.g. CookieReply(x, \"/callback?code=...&state=...&iss=...\") this is the path of the request that the apps get back when the user clicks approve. For this, apps need to register an app url or sorts. For the valid values for app URLs, see the redirect URIs for mobile platforms here https://git.sr.ht/~fkooman/vpn-user-portal/tree/v3/item/src/OAuth/VpnClientDb.php The client should open the webbrowser with this URL and continue the authorization process. This is only called if authorization needs to be retriggered Example Input (3=custom server): GetConfig(myCookie, 3, \"https://demo.eduvpn.nl/\", 0, 0) Example Output (2=WireGuard): { \"config\": \"[Interface]\\nPrivateKey = ...\\nAddress = ...\\nDNS = ...\\n\\n[Peer]\\nPublicKey = ...=\\nAllowedIPs = 0.0.0.0/0,::/0\\nEndpoint = ...\", \"protocol\": 2, \"default_gateway\": true, \"should_failover\": true, <- whether or not the failover procedure should happen }","title":"OAUTH_STARTED"},{"location":"apidocs.html#instate","text":"Signature: func InState(fsmState C.int) (C.int, *C.char) InState checks if the FSM is in fsmState . Example Input: InState(5) Example Output: 1, null","title":"InState"},{"location":"apidocs.html#newproxyguard","text":"Signature: func NewProxyguard(c C.uintptr_t, lp C.int, tcpsp C.int, peer *C.char, proxySetup C.ProxySetup) (C.uintptr_t, *C.char) NewProxyguard creates the \u2018proxyguard\u2019 procedure in eduvpn-common. If the proxy cannot be created it returns an error. This function proxies WireGuard UDP connections over HTTP: ProxyGuard on Codeberg . These input variables can be gotten from the configuration that is retrieved using the proxy JSON key c is the cookie. Note that if you cancel/delete the cookie, ProxyGuard gets cleaned up. Common automatically cleans up ProxyGuard when Cleanup is called, but it is good to cleanup yourself too. lp is the port of the local udp ProxyGuard connection, this is what is set to the WireGuard endpoint tcpsp is the TCP source port. Pass 0 if you do not route based on source port, so far only the Linux client has to pass non-zero. peer is the ip:port of the remote server proxySetup is a callback which is called when the socket is setting up, this can be used for configuring routing in the client. It takes two arguments: the file descriptor (integer) and a JSON list of IPs the client connects to Example Input: NewProxyguard(myCookie, 1337, 0, \"5.5.5.5:51820\", proxySetupCB) Example Output: null","title":"NewProxyguard"},{"location":"apidocs.html#proxyguardpeerips","text":"Signature: func ProxyguardPeerIPs(proxyH C.uintptr_t) (*C.char, *C.char) ProxyguardPeerIPs gets the Peer IPs configured by ProxyGuard Example Input: ProxyguardPeerIPs(handle) Example Output: [\"1.1.1.1\"], null","title":"ProxyguardPeerIPs"},{"location":"apidocs.html#proxyguardrestart","text":"Signature: func ProxyguardRestart(proxyH C.uintptr_t) *C.char ProxyguardRestart restarts ProxyGuard, call this when a network change happens Example Input: ProxyguardRestart(proxyHandle) Example Output: \"failed restarting ProxyGuard\"","title":"ProxyguardRestart"},{"location":"apidocs.html#proxyguardtunnel","text":"Signature: func ProxyguardTunnel(c C.uintptr_t, proxyH C.uintptr_t, wglisten C.int) *C.char ProxyguardTunnel starts the tunneling for ProxyGuard c is the cookie proxyH is the proxy handle wglisten is the port WireGuard is listening on","title":"ProxyguardTunnel"},{"location":"apidocs.html#register","text":"Signature: func Register( name *C.char, version *C.char, configDirectory *C.char, cb C.StateCB, ) *C.char Register creates a new client and also registers the FSM to go to the initial state Name is the name of the client, must be a valid client ID. Version is the version of the client. This version field is used for the user agent in all HTTP requests. cb is the state callback. It takes three arguments: The old state, the new state and the data for the state as JSON. Note that the states are defined in client/fsm.go, e.g. Main (in Go: StateMain ), ASK_PROFILE (in Go: StateAskProfile ) This callback returns non-zero if the state transition is handled. This is used to check if the client handles the needed transitions After registering, the FSM is initialized and the state transition MAIN should have been completed If some error occurs during registering, it is returned as a types/error/error.go Error Example Input: Register(\"org.eduvpn.app.linux\", \"0.0.1\", \"/tmp/eduvpn-common\", myCallbackFunc, 1) Example Output: { \"message\": { \"en\": \"failed to register, a VPN state is already present\" }, \"misc\": false }","title":"Register"},{"location":"apidocs.html#removeserver","text":"Signature: func RemoveServer(_type C.int, id *C.char) *C.char RemoveServer removes a server from the eduvpn-common server list _type is the type of server that needs to be added. This type is defined in types/server/server.go Type id is the identifier of the string: In case of secure internet: The organization ID In case of custom server: The base URL In case of institute access: The base URL If the server cannot be removed it returns the error types/error/error.go Error . Example Input (3=custom server): RemoveServer(3, \"bogus\") Example Output: { \"message\": { \"en\": \"failed to remove server\" }, \"misc\": false }","title":"RemoveServer"},{"location":"apidocs.html#renewsession","text":"Signature: func RenewSession(c C.uintptr_t) *C.char RenewSession renews the session of the VPN This essentially means that the OAuth tokens are deleted. And it also possibly re-runs every state callback you need when getting a config. So least you MUST handle the OAuth started transition It returns an error if unsuccessful. Example Input: RenewSession(myCookie) Example Output: { \"message\": { \"en\": \"could not renew session\" }, \"misc\": false }","title":"RenewSession"},{"location":"apidocs.html#serverlist","text":"Signature: func ServerList() (*C.char, *C.char) ServerList gets the list of servers that are currently added This is NOT the discovery list, but the servers that have previously been added with AddServer . It returns the server list as a JSON string defined in types/server/server.go List . If the server list cannot be retrieved it returns a nil string and an error. Example Input: ServerList() Example Output (current profile here is empty as none has been chosen yet): { \"institute_access_servers\": [ { \"display_name\": { \"en\": \"Demo\" }, \"identifier\": \"https://demo.eduvpn.nl/\", \"profiles\": { \"current\": \"\" }, \"support_contacts\": [ \"mailto:eduvpn@surf.nl\" ], \"delisted\": false } ] }, null","title":"ServerList"},{"location":"apidocs.html#setprofileid","text":"Signature: func SetProfileID(data *C.char) *C.char SetProfileID sets the profile ID of the current serrver. This MUST only be called if the user/client wishes to manually set a profile instead of the common lib asking for one using a transition. data is the profile ID. It returns an error if unsuccessful. Example Input: SetProfileID(\"splittunnel\") Example Output: { \"message\": { \"en\": \"profile does not exist\" }, \"misc\": false }","title":"SetProfileID"},{"location":"apidocs.html#setsecurelocation","text":"Signature: func SetSecureLocation(orgID *C.char, cc *C.char) *C.char SetSecureLocation sets the location for the secure internet server if it exists. This MUST only be called if the user/client wishes to manually set a location instead of the common lib asking for one using a transition. orgID is the organisation ID for the secure internet server cc is the location ID/country code It returns an error if unsuccessful. Example Input: SetSecureLocation(\"http://idp.geant.org/\", \"nl\") Example Output: { \"message\": { \"en\": \"location does not exist\" }, \"misc\": false }","title":"SetSecureLocation"},{"location":"apidocs.html#setstate","text":"Signature: func SetState(fsmState C.int) *C.char SetState sets the state of the state machine. Note: this transitions the FSM into the new state without passing any data to it. Example Input: SetState(5) Example Output: null","title":"SetState"},{"location":"apidocs.html#settokenhandler","text":"Signature: func SetTokenHandler(getter C.TokenGetter, setter C.TokenSetter) *C.char SetTokenHandler sets the token getters and token setters for OAuth. Because the data that is saved does not contain OAuth tokens for server, the common lib asks and sets the tokens using these callback functions. The client can thus pass callbacks to this function so that the tokens can be securely stored in a keyring. The client must pass two callback arguments to this function: getter is the void function that gets tokens from the client. It takes three arguments: - The server for which to get the tokens for, marshalled as JSON and defined in types/server/server.go Current - The output buffer - The length of the output buffer. This \u2018output buffer\u2019 must contain the tokens, marshalled as JSON that is defined in types/server/server.go Tokens setter is the void function that sets tokens. It takes two arguments: The server for which to get the tokens for, marshalled as JSON and defined in types/server/server.go Current The tokens , defined in types/server/server.go Tokens marshalled as JSON It returns an error when the tokens cannot be set. Example Input: SetTokenHandler(getterFunc, setterFunc) Example Output: null","title":"SetTokenHandler"},{"location":"apidocs.html#startfailover","text":"Signature: func StartFailover(c C.uintptr_t, gateway *C.char, mtu C.int, readRxBytes C.ReadRxBytes) (C.int, *C.char) StartFailover starts the \u2018failover\u2019 procedure in eduvpn-common. Failover has one primary goal: check if the VPN can reach the gateway. This can be used to check whether or not the client needs to \u2018failover\u2019 to prefer TCP (if currently using UDP). Which is useful to go from a broken WireGuard connection to OpenVPN over TCP. c is the cookie that is passed for cancellation. To create a cookie, use the CookieNew function gateway is the gateway IP of the VPN. You MAY calculate this with the CalculateGateway function readRxBytes is a function that returns the current rx bytes of the VPN interface, this should return a long long int in c It returns a boolean whether or not the common lib has determined that it cannot reach the gateway. Non-zero=dropped, zero=not dropped. It also returns an error, if it fails to indicate if it has dropped or not. In this case, dropped is also set to zero. Example Input: StartFailover(myCookie, \"10.10.10.1\", 1400, myRxBytesReader) Example Output: 1, null","title":"StartFailover"},{"location":"building-client.html","text":"Building a client # This chapter is a high-level overview on how to use eduvpn-common and build your own eduVPN/Let\u2019s Connect! client. In this chapter, we go over the basics of how the interop between Go and language x works, say something about the architecture, explain where to find detailed API documentation, explain the state machine, give a typical flow for a client and give a follow along tutorial on building an eduVPN client using Python code. At last, we will also have a few code examples that can be used as a short reference. Go <-> language X interop # Because this library is meant to be a general library for other clients to use that are written in different programming languages, we need to find a way to make this Go library available on each platform and codebase. The approach that we take is to build a C library from the Go library using Cgo. Cgo can have its disadvantages with performance and the constant conversion between Go and C types. To overcome those barriers, this library has the following goals (with some others noted here): Be high-level . Functions should do as much as possible in Go. The exported API should fit in one file. Lots of low-level functions would be a constant conversion between C and Go which adds overhead Move as much state to Go as possible . For example, Go keeps track of the servers you have configured and discovery. This makes the arguments to functions simple, clients should pass simple identifiers that Go can look up in the state Easy type conversion : to convert between C and Go types, JSON is used. Whereas Protobuf, Cap\u2019n\u2019proto or flatbuffers are more performant, they are harder to debug, add thousands of lines of autogenerated code and are not human friendly. Using JSON, the clients can approach it the same way they would use with a server using a REST API. Another approach is to just convert from Go -> C types -> language types. This was tried in version 1 of the library, but this ended up being too much work and manual memory management Make it as easy as possible for clients to manage UI and internal state : we use a state machine that gives the clients information in which state the Go library is in, e.g. we\u2019re selecting a server profile, we\u2019re loading the server endpoints. This library is not only a layer to talk to eduVPN servers, but the whole engine for a client Implement features currently not present in existing clients : WireGuard to OpenVPN failover, WireGuard over TCP Follow the official eduVPN specification and also contribute changes when needed Secure : We aim to follow the latest OAuth recommendations, to not store secret data and e.g. disable OpenVPN scripts from being ran by default And finally the most important goal: The advantages that this library brings for clients should outweigh the cost of incorporating it into the codebase . Initial versions would take more work than we get out of it. However, when each eduVPN/Let\u2019s Connect! client uses this library we should expect a net gain. New features should be easier to implement for clients by simply requiring a new eduvpn-common version and using the necessary functions Architecture # In the previous section, we have already hinted a bit on the exact architecture. This section will expand upon it by giving a figure of the basic structure As can be seen by this architecture, there is an intermediate layer between the client and the shared library. This wrapper eases the way of loading this library and then defining a more language specific API for it. In the eduvpn-common repo, we currently only support a Python wrapper. Clients themselves can define their own wrapper Typical flow for a client # NOTE: This uses the function names that are defined in the exports file in Go. For your own wrapper/the Python wrapper they are different. But the general flow is the same The client starts up. It calls the Register function that communicates with the library that it has initialized It gets the list of servers using ServerList When the user selects a server to connect to in the UI, it calls the GetConfig to get a VPN configuration for this server. This function transitions the state machine multiple times. The client uses these state transitions for logging or even updating the UI. The client then connects New feature in eduvpn-common: Check if the VPN can reach the gateway after the client is connected by calling StartFailover If the client has no servers, or it wants to add a new server, the client calls DiscoOrganizations and DiscoServers to get the discovery files from the library. This even returns cached copies if the organizations or servers should not have been updated according to the documentation From this discovery list, it calls AddServer to add the server to the internal server list of eduvpn-common. This also calls necessary state transitions, e.g. for authorizing the server. The next call to ServerList then has this server included It can then get a configuration for this server like we have explained in step 3 When a configuration has been obtained, the internal state has changed and the client can get the current server that was configured using CurrentServer . CurrentServer can also be called after startup if a server was previously set as the current server When the VPN disconnects, the client calls Cleanup so that the server resources are cleaned up by calling the /disconnect endpoint A server can be removed with the RemoveServer function When the client is done, it calls Deregister such that the most up to date internal state is saved to disk. Note that eduvpn-common also saves the internal state .e.g. after obtaining a VPN configuration Finite state machine # The eduvpn-common library uses a finite state machine internally to keep track of which state the client is in and to communicate data callbacks (e.g. to communicate the Authorization URL in the OAuth process to the client). FSM example # The following is an example of the FSM when the client has obtained a Wireguard/OpenVPN configuration from an eduVPN server The current state is highlighted in the cyan color. State explanation # For the explanation of what all the different states mean, see the client documentation States that ask data # In eduvpn-common, there are certain states that require attention from the client. OAuth Started: A state that must be handled by the client. How a client can \u2018handle\u2019 this state, we will see in the next section. In this state, the client must open the webbrowser with the authorization URL to complete to OAuth process. Note that on mobile platforms, you also need to reply with the authorization URI as these platforms do not support a local callback server using 127.0.0.1 Ask Profile: The state that asks for a profile selection to the client. Reply to this state by using a \u201ccookie\u201d and the CookieReply function. What this means will be discussed in the Python client example too Ask Location: Same for ask profile but for selecting a secure internet location. Only called if one must be chosen, e.g. due to a selection that is no longer valid The rest of the states are miscellaneous states, meaning that the client can handle them however it wants to. However, it can be useful to handle most state transitions to e.g. show loading screens or for logging and debugging purposes. Code examples # This chapter contains code examples that use the API Go command line client # The following is an example in the repository . It is a command line client. {// Package main implements an example CLI client package main import ( \"context\" \"flag\" \"fmt\" \"os\" \"reflect\" \"strings\" \"codeberg.org/eduVPN/eduvpn-common/client\" \"codeberg.org/eduVPN/eduvpn-common/i18n\" \"codeberg.org/eduVPN/eduvpn-common/internal/commonver\" \"codeberg.org/eduVPN/eduvpn-common/types/cookie\" srvtypes \"codeberg.org/eduVPN/eduvpn-common/types/server\" \"github.com/pkg/browser\" ) // Open a browser with xdg-open. func openBrowser(data any) { str, ok := data.(string) if !ok { return } go func() { err := browser.OpenURL(str) if err != nil { fmt.Fprintln(os.Stderr, \"failed to open browser with error:\", err) fmt.Println(\"Please open your browser manually\") } }() } func getProfileInteractive(profiles *srvtypes.Profiles, data any) (string, error) { fmt.Printf(\"Multiple VPN profiles found. Please select a profile by entering e.g. 1\") var ps strings.Builder var options []string i := 0 for k, v := range profiles.Map { fmt.Fprintf(&ps, \"\\n%d - %s\", i+1, i18n.GetLanguageMatched(v.DisplayName, \"en\")) options = append(options, k) i++ } // Show the profiles fmt.Println(ps.String()) var idx int if _, err := fmt.Scanf(\"%d\", &idx); err != nil || idx <= 0 || idx > len(profiles.Map) { fmt.Fprintln(os.Stderr, \"invalid profile chosen, please retry\") return getProfileInteractive(profiles, data) } p := options[idx-1] fmt.Println(\"Sending profile ID\", p) return p, nil } func sendProfile(profile string, data any) { d, ok := data.(*srvtypes.RequiredAskTransition) if !ok { fmt.Fprintf(os.Stderr, \"\\ninvalid data type: %v\\n\", reflect.TypeOf(data)) os.Exit(1) } sps, ok := d.Data.(*srvtypes.Profiles) if !ok { fmt.Fprintf(os.Stderr, \"\\ninvalid data type for profiles: %v\\n\", reflect.TypeOf(d.Data)) os.Exit(1) } if profile == \"\" { gprof, err := getProfileInteractive(sps, data) if err != nil { fmt.Fprintf(os.Stderr, \"failed getting profile interactively: %v\\n\", err) os.Exit(1) } profile = gprof } if err := d.C.Send(profile); err != nil { fmt.Fprintf(os.Stderr, \"failed setting profile with error: %v\\n\", err) os.Exit(1) } } // The callback function // If OAuth is started we open the browser with the Auth URL // If we ask for a profile, we send the profile using command line input // Note that this has an additional argument, the vpn state which was wrapped into this callback function below. func stateCallback(_ client.FSMStateID, newState client.FSMStateID, data any, prof string, dir string) { if newState == client.StateOAuthStarted { openBrowser(data) } if newState == client.StateAskProfile { sendProfile(prof, data) } if newState == client.StateAskLocation { // removing is best effort _ = os.RemoveAll(dir) fmt.Fprint(os.Stderr, \"An invalid secure location is stored. This CLI doesn't support interactively choosing a location yet. Give a correct location with the -country-code flag\") os.Exit(1) } } // Get a config for Institute Access or Secure Internet Server. func getConfig(state *client.Client, url string, srvType srvtypes.Type, cc string, prof string) (*srvtypes.Configuration, error) { if !strings.HasPrefix(url, \"http\") { url = \"https://\" + url } ck := cookie.NewWithContext(context.Background()) defer ck.Cancel() //nolint:errcheck err := state.AddServer(ck, url, srvType, nil) if err != nil { // TODO: This is quite hacky :^) if !strings.Contains(err.Error(), \"a secure internet server already exists.\") { return nil, err } } if cc != \"\" { err = state.SetSecureLocation(url, cc) if err != nil { return nil, err } } if prof != \"\" { // this is best effort, e.g. if no server was chosen before this fails _ = state.SetProfileID(prof) //nolint:errcheck } return state.GetConfig(ck, url, srvType, false, false) } // Get a config for a single server, Institute Access or Secure Internet. func printConfig(url string, cc string, srvType srvtypes.Type, prof string) error { var c *client.Client var err error var dir string dir, err = os.MkdirTemp(\"\", \"eduvpn-common\") if err != nil { return err } // removing is best effort defer os.RemoveAll(dir) //nolint:errcheck c, err = client.New( \"org.eduvpn.app.linux\", fmt.Sprintf(\"%s-cli\", commonver.Version), dir, func(oldState client.FSMStateID, newState client.FSMStateID, data any) bool { stateCallback(oldState, newState, data, prof, dir) return true }, nil, ) if err != nil { return err } _ = c.Register() ck := cookie.NewWithContext(context.Background()) _, err = c.DiscoOrganizations(ck, false, \"\") if err != nil { return err } _, err = c.DiscoServers(ck, false, \"\") if err != nil { return err } defer c.Deregister() cfg, err := getConfig(c, url, srvType, cc, prof) if err != nil { return err } fmt.Println(cfg.VPNConfig) return nil } // The main function // It parses the arguments and executes the correct functions. func main() { cu := flag.String(\"get-custom\", \"\", \"The url of a custom server to connect to\") u := flag.String(\"get-institute\", \"\", \"The url of an institute to connect to\") sec := flag.String(\"get-secure\", \"\", \"Gets secure internet servers\") cc := flag.String(\"country-code\", \"\", \"The country code to use in case of a secure internet server\") prof := flag.String(\"profile\", \"\", \"The profile ID to choose\") flag.Parse() // Connect to a VPN by getting an Institute Access config var err error switch { case *cu != \"\": err = printConfig(*cu, \"\", srvtypes.TypeCustom, *prof) case *u != \"\": err = printConfig(*u, \"\", srvtypes.TypeInstituteAccess, *prof) case *sec != \"\": err = printConfig(*sec, *cc, srvtypes.TypeSecureInternet, *prof) default: flag.PrintDefaults() } if err != nil { fmt.Fprintf(os.Stderr, \"failed to get a VPN config: %v\\n\", err) } }}","title":"Building a client"},{"location":"building-client.html#go-language-x-interop","text":"Because this library is meant to be a general library for other clients to use that are written in different programming languages, we need to find a way to make this Go library available on each platform and codebase. The approach that we take is to build a C library from the Go library using Cgo. Cgo can have its disadvantages with performance and the constant conversion between Go and C types. To overcome those barriers, this library has the following goals (with some others noted here): Be high-level . Functions should do as much as possible in Go. The exported API should fit in one file. Lots of low-level functions would be a constant conversion between C and Go which adds overhead Move as much state to Go as possible . For example, Go keeps track of the servers you have configured and discovery. This makes the arguments to functions simple, clients should pass simple identifiers that Go can look up in the state Easy type conversion : to convert between C and Go types, JSON is used. Whereas Protobuf, Cap\u2019n\u2019proto or flatbuffers are more performant, they are harder to debug, add thousands of lines of autogenerated code and are not human friendly. Using JSON, the clients can approach it the same way they would use with a server using a REST API. Another approach is to just convert from Go -> C types -> language types. This was tried in version 1 of the library, but this ended up being too much work and manual memory management Make it as easy as possible for clients to manage UI and internal state : we use a state machine that gives the clients information in which state the Go library is in, e.g. we\u2019re selecting a server profile, we\u2019re loading the server endpoints. This library is not only a layer to talk to eduVPN servers, but the whole engine for a client Implement features currently not present in existing clients : WireGuard to OpenVPN failover, WireGuard over TCP Follow the official eduVPN specification and also contribute changes when needed Secure : We aim to follow the latest OAuth recommendations, to not store secret data and e.g. disable OpenVPN scripts from being ran by default And finally the most important goal: The advantages that this library brings for clients should outweigh the cost of incorporating it into the codebase . Initial versions would take more work than we get out of it. However, when each eduVPN/Let\u2019s Connect! client uses this library we should expect a net gain. New features should be easier to implement for clients by simply requiring a new eduvpn-common version and using the necessary functions","title":"Go <-> language X interop"},{"location":"building-client.html#architecture","text":"In the previous section, we have already hinted a bit on the exact architecture. This section will expand upon it by giving a figure of the basic structure As can be seen by this architecture, there is an intermediate layer between the client and the shared library. This wrapper eases the way of loading this library and then defining a more language specific API for it. In the eduvpn-common repo, we currently only support a Python wrapper. Clients themselves can define their own wrapper","title":"Architecture"},{"location":"building-client.html#typical-flow-for-a-client","text":"NOTE: This uses the function names that are defined in the exports file in Go. For your own wrapper/the Python wrapper they are different. But the general flow is the same The client starts up. It calls the Register function that communicates with the library that it has initialized It gets the list of servers using ServerList When the user selects a server to connect to in the UI, it calls the GetConfig to get a VPN configuration for this server. This function transitions the state machine multiple times. The client uses these state transitions for logging or even updating the UI. The client then connects New feature in eduvpn-common: Check if the VPN can reach the gateway after the client is connected by calling StartFailover If the client has no servers, or it wants to add a new server, the client calls DiscoOrganizations and DiscoServers to get the discovery files from the library. This even returns cached copies if the organizations or servers should not have been updated according to the documentation From this discovery list, it calls AddServer to add the server to the internal server list of eduvpn-common. This also calls necessary state transitions, e.g. for authorizing the server. The next call to ServerList then has this server included It can then get a configuration for this server like we have explained in step 3 When a configuration has been obtained, the internal state has changed and the client can get the current server that was configured using CurrentServer . CurrentServer can also be called after startup if a server was previously set as the current server When the VPN disconnects, the client calls Cleanup so that the server resources are cleaned up by calling the /disconnect endpoint A server can be removed with the RemoveServer function When the client is done, it calls Deregister such that the most up to date internal state is saved to disk. Note that eduvpn-common also saves the internal state .e.g. after obtaining a VPN configuration","title":"Typical flow for a client"},{"location":"building-client.html#finite-state-machine","text":"The eduvpn-common library uses a finite state machine internally to keep track of which state the client is in and to communicate data callbacks (e.g. to communicate the Authorization URL in the OAuth process to the client).","title":"Finite state machine"},{"location":"building-client.html#fsm-example","text":"The following is an example of the FSM when the client has obtained a Wireguard/OpenVPN configuration from an eduVPN server The current state is highlighted in the cyan color.","title":"FSM example"},{"location":"building-client.html#state-explanation","text":"For the explanation of what all the different states mean, see the client documentation","title":"State explanation"},{"location":"building-client.html#states-that-ask-data","text":"In eduvpn-common, there are certain states that require attention from the client. OAuth Started: A state that must be handled by the client. How a client can \u2018handle\u2019 this state, we will see in the next section. In this state, the client must open the webbrowser with the authorization URL to complete to OAuth process. Note that on mobile platforms, you also need to reply with the authorization URI as these platforms do not support a local callback server using 127.0.0.1 Ask Profile: The state that asks for a profile selection to the client. Reply to this state by using a \u201ccookie\u201d and the CookieReply function. What this means will be discussed in the Python client example too Ask Location: Same for ask profile but for selecting a secure internet location. Only called if one must be chosen, e.g. due to a selection that is no longer valid The rest of the states are miscellaneous states, meaning that the client can handle them however it wants to. However, it can be useful to handle most state transitions to e.g. show loading screens or for logging and debugging purposes.","title":"States that ask data"},{"location":"building-client.html#code-examples","text":"This chapter contains code examples that use the API","title":"Code examples"},{"location":"building-client.html#go-command-line-client","text":"The following is an example in the repository . It is a command line client. {// Package main implements an example CLI client package main import ( \"context\" \"flag\" \"fmt\" \"os\" \"reflect\" \"strings\" \"codeberg.org/eduVPN/eduvpn-common/client\" \"codeberg.org/eduVPN/eduvpn-common/i18n\" \"codeberg.org/eduVPN/eduvpn-common/internal/commonver\" \"codeberg.org/eduVPN/eduvpn-common/types/cookie\" srvtypes \"codeberg.org/eduVPN/eduvpn-common/types/server\" \"github.com/pkg/browser\" ) // Open a browser with xdg-open. func openBrowser(data any) { str, ok := data.(string) if !ok { return } go func() { err := browser.OpenURL(str) if err != nil { fmt.Fprintln(os.Stderr, \"failed to open browser with error:\", err) fmt.Println(\"Please open your browser manually\") } }() } func getProfileInteractive(profiles *srvtypes.Profiles, data any) (string, error) { fmt.Printf(\"Multiple VPN profiles found. Please select a profile by entering e.g. 1\") var ps strings.Builder var options []string i := 0 for k, v := range profiles.Map { fmt.Fprintf(&ps, \"\\n%d - %s\", i+1, i18n.GetLanguageMatched(v.DisplayName, \"en\")) options = append(options, k) i++ } // Show the profiles fmt.Println(ps.String()) var idx int if _, err := fmt.Scanf(\"%d\", &idx); err != nil || idx <= 0 || idx > len(profiles.Map) { fmt.Fprintln(os.Stderr, \"invalid profile chosen, please retry\") return getProfileInteractive(profiles, data) } p := options[idx-1] fmt.Println(\"Sending profile ID\", p) return p, nil } func sendProfile(profile string, data any) { d, ok := data.(*srvtypes.RequiredAskTransition) if !ok { fmt.Fprintf(os.Stderr, \"\\ninvalid data type: %v\\n\", reflect.TypeOf(data)) os.Exit(1) } sps, ok := d.Data.(*srvtypes.Profiles) if !ok { fmt.Fprintf(os.Stderr, \"\\ninvalid data type for profiles: %v\\n\", reflect.TypeOf(d.Data)) os.Exit(1) } if profile == \"\" { gprof, err := getProfileInteractive(sps, data) if err != nil { fmt.Fprintf(os.Stderr, \"failed getting profile interactively: %v\\n\", err) os.Exit(1) } profile = gprof } if err := d.C.Send(profile); err != nil { fmt.Fprintf(os.Stderr, \"failed setting profile with error: %v\\n\", err) os.Exit(1) } } // The callback function // If OAuth is started we open the browser with the Auth URL // If we ask for a profile, we send the profile using command line input // Note that this has an additional argument, the vpn state which was wrapped into this callback function below. func stateCallback(_ client.FSMStateID, newState client.FSMStateID, data any, prof string, dir string) { if newState == client.StateOAuthStarted { openBrowser(data) } if newState == client.StateAskProfile { sendProfile(prof, data) } if newState == client.StateAskLocation { // removing is best effort _ = os.RemoveAll(dir) fmt.Fprint(os.Stderr, \"An invalid secure location is stored. This CLI doesn't support interactively choosing a location yet. Give a correct location with the -country-code flag\") os.Exit(1) } } // Get a config for Institute Access or Secure Internet Server. func getConfig(state *client.Client, url string, srvType srvtypes.Type, cc string, prof string) (*srvtypes.Configuration, error) { if !strings.HasPrefix(url, \"http\") { url = \"https://\" + url } ck := cookie.NewWithContext(context.Background()) defer ck.Cancel() //nolint:errcheck err := state.AddServer(ck, url, srvType, nil) if err != nil { // TODO: This is quite hacky :^) if !strings.Contains(err.Error(), \"a secure internet server already exists.\") { return nil, err } } if cc != \"\" { err = state.SetSecureLocation(url, cc) if err != nil { return nil, err } } if prof != \"\" { // this is best effort, e.g. if no server was chosen before this fails _ = state.SetProfileID(prof) //nolint:errcheck } return state.GetConfig(ck, url, srvType, false, false) } // Get a config for a single server, Institute Access or Secure Internet. func printConfig(url string, cc string, srvType srvtypes.Type, prof string) error { var c *client.Client var err error var dir string dir, err = os.MkdirTemp(\"\", \"eduvpn-common\") if err != nil { return err } // removing is best effort defer os.RemoveAll(dir) //nolint:errcheck c, err = client.New( \"org.eduvpn.app.linux\", fmt.Sprintf(\"%s-cli\", commonver.Version), dir, func(oldState client.FSMStateID, newState client.FSMStateID, data any) bool { stateCallback(oldState, newState, data, prof, dir) return true }, nil, ) if err != nil { return err } _ = c.Register() ck := cookie.NewWithContext(context.Background()) _, err = c.DiscoOrganizations(ck, false, \"\") if err != nil { return err } _, err = c.DiscoServers(ck, false, \"\") if err != nil { return err } defer c.Deregister() cfg, err := getConfig(c, url, srvType, cc, prof) if err != nil { return err } fmt.Println(cfg.VPNConfig) return nil } // The main function // It parses the arguments and executes the correct functions. func main() { cu := flag.String(\"get-custom\", \"\", \"The url of a custom server to connect to\") u := flag.String(\"get-institute\", \"\", \"The url of an institute to connect to\") sec := flag.String(\"get-secure\", \"\", \"Gets secure internet servers\") cc := flag.String(\"country-code\", \"\", \"The country code to use in case of a secure internet server\") prof := flag.String(\"profile\", \"\", \"The profile ID to choose\") flag.Parse() // Connect to a VPN by getting an Institute Access config var err error switch { case *cu != \"\": err = printConfig(*cu, \"\", srvtypes.TypeCustom, *prof) case *u != \"\": err = printConfig(*u, \"\", srvtypes.TypeInstituteAccess, *prof) case *sec != \"\": err = printConfig(*sec, *cc, srvtypes.TypeSecureInternet, *prof) default: flag.PrintDefaults() } if err != nil { fmt.Fprintf(os.Stderr, \"failed to get a VPN config: %v\\n\", err) } }}","title":"Go command line client"},{"location":"building.html","text":"Building # This section contains the instruction on how to build the library and associated wrappers. We first explain how to build the Go library and then further explain the wrapper specific building process. As the Python wrapper is the only wrapper at the moment, this only consists of this wrapper language for now. Building the Go library # To build the Go library, you need the dependencies for your system installed. We will go over the needed dependencies for Linux. Afterwards, we explain the basic commands to build the library. Dependencies # Linux # To build the Go shared library using Linux you need the following dependencies: Go 1.18 or later Gcc GNU Make Dependencies for the Python wrapper if you want to build that as well Commands # Before we can begin building the wrapper code, we need to build the Go code as a shared library. This section will tell you how to do so. To build the shared library for the current platform issue the following command in the root directory: make The shared library will be output in lib/ . Cleaning # To clean build the library and wrapper, issue the following command in the root directory: make clean Python wrapper # To build the python wrapper issue the following command (in the root directory of the eduvpn-common project): make -C wrappers/python This uses the makefile in wrappers/python/Makefile to build the python file into a wheel placed in wrappers/python/dist/eduvpncommon-[version]-py3-none-[platform].whl . Where version is the version of the library and platform is your current platform. The wheel can be installed with pip : pip install ./wrappers/python/dist/eduvpncommon-[version]-py3-none-[platform].whl Notes on building for release # To build for release, make sure you obtain the tarball artifacts in the release (ending with .tar.xz ) at https://codeberg.org/eduVPN/eduvpn-common/releases . These are signed with minisign and gpg keys, make sure to verify these signatures using the public keys available here: https://codeberg.org/eduVPN/eduvpn-common/src/branch/main/keys , they are also available externally: - https://app.eduvpn.org/linux/v4/deb/app+linux@eduvpn.org.asc - https://git.sr.ht/~jwijenbergh/python3-eduvpn-common.rpm/tree/main/item/SOURCES/minisign-CA9409316AC93C07.pub To build for release, make sure to extract the tarball, and then build: make To upload the releases to Codeberg, run: ./make_release.sh For pre-releases: ./make_release.sh -p Package formats # We support the following additional package formats: RPM (Linux, Fedora), Deb (Linux, Debian derivatives) and Pip. These are build in the CI and for production build on a separate non-public host.","title":"Building"},{"location":"building.html#building-the-go-library","text":"To build the Go library, you need the dependencies for your system installed. We will go over the needed dependencies for Linux. Afterwards, we explain the basic commands to build the library.","title":"Building the Go library"},{"location":"building.html#dependencies","text":"","title":"Dependencies"},{"location":"building.html#commands","text":"Before we can begin building the wrapper code, we need to build the Go code as a shared library. This section will tell you how to do so. To build the shared library for the current platform issue the following command in the root directory: make The shared library will be output in lib/ .","title":"Commands"},{"location":"building.html#python-wrapper","text":"To build the python wrapper issue the following command (in the root directory of the eduvpn-common project): make -C wrappers/python This uses the makefile in wrappers/python/Makefile to build the python file into a wheel placed in wrappers/python/dist/eduvpncommon-[version]-py3-none-[platform].whl . Where version is the version of the library and platform is your current platform. The wheel can be installed with pip : pip install ./wrappers/python/dist/eduvpncommon-[version]-py3-none-[platform].whl","title":"Python wrapper"},{"location":"building.html#notes-on-building-for-release","text":"To build for release, make sure you obtain the tarball artifacts in the release (ending with .tar.xz ) at https://codeberg.org/eduVPN/eduvpn-common/releases . These are signed with minisign and gpg keys, make sure to verify these signatures using the public keys available here: https://codeberg.org/eduVPN/eduvpn-common/src/branch/main/keys , they are also available externally: - https://app.eduvpn.org/linux/v4/deb/app+linux@eduvpn.org.asc - https://git.sr.ht/~jwijenbergh/python3-eduvpn-common.rpm/tree/main/item/SOURCES/minisign-CA9409316AC93C07.pub To build for release, make sure to extract the tarball, and then build: make To upload the releases to Codeberg, run: ./make_release.sh For pre-releases: ./make_release.sh -p","title":"Notes on building for release"},{"location":"building.html#package-formats","text":"We support the following additional package formats: RPM (Linux, Fedora), Deb (Linux, Debian derivatives) and Pip. These are build in the CI and for production build on a separate non-public host.","title":"Package formats"},{"location":"testing.html","text":"Testing # The Go library right now has various tests defined. E.g. server interaction, oauth, discovery and signature verification tests. To run the Go test suite, issue the following command in a shell make test Note that this runs the tests without any server interaction (so for now only the signature verification tests). To run the tests with an eduVPN server you need to specify environment variables: SERVER_URI=\"eduvpn.example.com\" PORTAL_USER=\"example\" PORTAL_PASS=\"example\" make test This needs python3-selenium and geckodriver (extract and put in your $PATH ). Note that testing with a server assumes it uses a default portal, due to it needing to click on buttons on the web page. You can add your own portal by customizing the called Selenium script . There are other environment variables that can be used: OAUTH_EXPIRED_TTL : Use this for a server which has a low OAuth access token expiry time, e.g. 10 seconds. You would then set this variable to \"10\" so that a test is ran which waits for 10 seconds for the OAuth tokens to expire Testing the Python code # To test the Python code, issue the following command in a shell (you will need dependencies for all wrappers if you do this[^1]): make -C wrappers/python test","title":"Testing"},{"location":"testing.html#testing-the-python-code","text":"To test the Python code, issue the following command in a shell (you will need dependencies for all wrappers if you do this[^1]): make -C wrappers/python test","title":"Testing the Python code"}]}
\ No newline at end of file diff --git a/search/worker.js b/search/worker.js new file mode 100644 index 0000000..8628dbc --- /dev/null +++ b/search/worker.js @@ -0,0 +1,133 @@ +var base_path = 'function' === typeof importScripts ? '.' : '/search/'; +var allowSearch = false; +var index; +var documents = {}; +var lang = ['en']; +var data; + +function getScript(script, callback) { + console.log('Loading script: ' + script); + $.getScript(base_path + script).done(function () { + callback(); + }).fail(function (jqxhr, settings, exception) { + console.log('Error: ' + exception); + }); +} + +function getScriptsInOrder(scripts, callback) { + if (scripts.length === 0) { + callback(); + return; + } + getScript(scripts[0], function() { + getScriptsInOrder(scripts.slice(1), callback); + }); +} + +function loadScripts(urls, callback) { + if( 'function' === typeof importScripts ) { + importScripts.apply(null, urls); + callback(); + } else { + getScriptsInOrder(urls, callback); + } +} + +function onJSONLoaded () { + data = JSON.parse(this.responseText); + var scriptsToLoad = ['lunr.js']; + if (data.config && data.config.lang && data.config.lang.length) { + lang = data.config.lang; + } + if (lang.length > 1 || lang[0] !== "en") { + scriptsToLoad.push('lunr.stemmer.support.js'); + if (lang.length > 1) { + scriptsToLoad.push('lunr.multi.js'); + } + if (lang.includes("ja") || lang.includes("jp")) { + scriptsToLoad.push('tinyseg.js'); + } + for (var i=0; i < lang.length; i++) { + if (lang[i] != 'en') { + scriptsToLoad.push(['lunr', lang[i], 'js'].join('.')); + } + } + } + loadScripts(scriptsToLoad, onScriptsLoaded); +} + +function onScriptsLoaded () { + console.log('All search scripts loaded, building Lunr index...'); + if (data.config && data.config.separator && data.config.separator.length) { + lunr.tokenizer.separator = new RegExp(data.config.separator); + } + + if (data.index) { + index = lunr.Index.load(data.index); + data.docs.forEach(function (doc) { + documents[doc.location] = doc; + }); + console.log('Lunr pre-built index loaded, search ready'); + } else { + index = lunr(function () { + if (lang.length === 1 && lang[0] !== "en" && lunr[lang[0]]) { + this.use(lunr[lang[0]]); + } else if (lang.length > 1) { + this.use(lunr.multiLanguage.apply(null, lang)); // spread operator not supported in all browsers: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/Spread_operator#Browser_compatibility + } + this.field('title'); + this.field('text'); + this.ref('location'); + + for (var i=0; i < data.docs.length; i++) { + var doc = data.docs[i]; + this.add(doc); + documents[doc.location] = doc; + } + }); + console.log('Lunr index built, search ready'); + } + allowSearch = true; + postMessage({config: data.config}); + postMessage({allowSearch: allowSearch}); +} + +function init () { + var oReq = new XMLHttpRequest(); + oReq.addEventListener("load", onJSONLoaded); + var index_path = base_path + '/search_index.json'; + if( 'function' === typeof importScripts ){ + index_path = 'search_index.json'; + } + oReq.open("GET", index_path); + oReq.send(); +} + +function search (query) { + if (!allowSearch) { + console.error('Assets for search still loading'); + return; + } + + var resultDocuments = []; + var results = index.search(query); + for (var i=0; i < results.length; i++){ + var result = results[i]; + doc = documents[result.ref]; + doc.summary = doc.text.substring(0, 200); + resultDocuments.push(doc); + } + return resultDocuments; +} + +if( 'function' === typeof importScripts ) { + onmessage = function (e) { + if (e.data.init) { + init(); + } else if (e.data.query) { + postMessage({ results: search(e.data.query) }); + } else { + console.error("Worker - Unrecognized message: " + e); + } + }; +} |