2190 lines (1788 loc) · 171 KB

String Regex Annotator Tutorial

The StringRegexAnnotator makes it extremely easy to match several complex regular expressions against a document and annotate the matches and/or the part of a match corresponding to a capturing regular expression group.

It also has a simple macro substitution feature that makes it easy to build more complex regular expression from simpler ones.

import os
from gatenlp import Document
from gatenlp.processing.gazetteer import StringRegexAnnotator, StringGazetteer

Creating the Annotator

Similar to the gazetteer annotators, there are several ways of how the annotator can be created: from a file that contains the regular expression rules, from a string (consisting of several lines) that contains regular expression rules (basically the content of a file as a string) or from prepared rule objects. Which of this to use is specified with the source_fmt parameter of either the constructor or the append method.

Create from a string with rules

The following example shows a string that contains a single simple rule which finds a date in ISO format (YYYY-MM-DD) and annotates it with annotation type "Date"

rules1 = """
|[0-9]{4}-[0-9]{2}-[0-9]{2}
0 => Date
"""

annt1 = StringRegexAnnotator(source=rules1, source_fmt="string")

doc1 = Document("A document that contains a date here: 2013-01-12 and also here: 1999-12-31")

annt1(doc1)
doc1

"],col:[2,"","

"],tr:[2,"","

"],td:[3,"","

"],_default:[0,"",""]};function ve(e,t){var n;return n="undefined"!=typeof e.getElementsByTagName?e.getElementsByTagName(t||"*"):"undefined"!=typeof e.querySelectorAll?e.querySelectorAll(t||"*"):[],void 0===t||t&&A(e,t)?S.merge([e],n):n}function ye(e,t){for(var n=0,r=e.length;n",""]);var me=/<|&#?\w+;/;function xe(e,t,n,r,i){for(var o,a,s,u,l,c,f=t.createDocumentFragment(),p=[],d=0,h=e.length;d\s*$/g;function qe(e,t){return A(e,"table")&&A(11!==t.nodeType?t:t.firstChild,"tr")&&S(e).children("tbody")[0]||e}function Le(e){return e.type=(null!==e.getAttribute("type"))+"/"+e.type,e}function He(e){return"true/"===(e.type||"").slice(0,5)?e.type=e.type.slice(5):e.removeAttribute("type"),e}function Oe(e,t){var n,r,i,o,a,s;if(1===t.nodeType){if(Y.hasData(e)&&(s=Y.get(e).events))for(i in Y.remove(t,"handle events"),s)for(n=0,r=s[i].length;n").attr(n.scriptAttrs||{}).prop({charset:n.scriptCharset,src:n.url}).on("load error",i=function(e){r.remove(),i=null,e&&t("error"===e.type?404:200,e.type)}),E.head.appendChild(r[0])},abort:function(){i&&i()}}});var Ut,Xt=[],Vt=/(=)\?(?=&|$)|\?\?/;S.ajaxSetup({jsonp:"callback",jsonpCallback:function(){var e=Xt.pop()||S.expando+"_"+Ct.guid++;return this[e]=!0,e}}),S.ajaxPrefilter("json jsonp",function(e,t,n){var r,i,o,a=!1!==e.jsonp&&(Vt.test(e.url)?"url":"string"==typeof e.data&&0===(e.contentType||"").indexOf("application/x-www-form-urlencoded")&&Vt.test(e.data)&&"data");if(a||"jsonp"===e.dataTypes[0])return r=e.jsonpCallback=m(e.jsonpCallback)?e.jsonpCallback():e.jsonpCallback,a?e[a]=e[a].replace(Vt,"$1"+r):!1!==e.jsonp&&(e.url+=(Et.test(e.url)?"&":"?")+e.jsonp+"="+r),e.converters["script json"]=function(){return o||S.error(r+" was not called"),o[0]},e.dataTypes[0]="json",i=C[r],C[r]=function(){o=arguments},n.always(function(){void 0===i?S(C).removeProp(r):C[r]=i,e[r]&&(e.jsonpCallback=t.jsonpCallback,Xt.push(r)),o&&m(i)&&i(o[0]),o=i=void 0}),"script"}),y.createHTMLDocument=((Ut=E.implementation.createHTMLDocument("").body).innerHTML="",2===Ut.childNodes.length),S.parseHTML=function(e,t,n){return"string"!=typeof e?[]:("boolean"==typeof t&&(n=t,t=!1),t||(y.createHTMLDocument?((r=(t=E.implementation.createHTMLDocument("")).createElement("base")).href=E.location.href,t.head.appendChild(r)):t=E),o=!n&&[],(i=N.exec(e))?[t.createElement(i[1])]:(i=xe([e],t,o),o&&o.length&&S(o).remove(),S.merge([],i.childNodes)));var r,i,o},S.fn.load=function(e,t,n){var r,i,o,a=this,s=e.indexOf(" ");return-1").append(S.parseHTML(e)).find(r):e)}).always(n&&function(e,t){a.each(function(){n.apply(this,o||[e.responseText,t,e])})}),this},S.expr.pseudos.animated=function(t){return S.grep(S.timers,function(e){return t===e.elem}).length},S.offset={setOffset:function(e,t,n){var r,i,o,a,s,u,l=S.css(e,"position"),c=S(e),f={};"static"===l&&(e.style.position="relative"),s=c.offset(),o=S.css(e,"top"),u=S.css(e,"left"),("absolute"===l||"fixed"===l)&&-1<(o+u).indexOf("auto")?(a=(r=c.position()).top,i=r.left):(a=parseFloat(o)||0,i=parseFloat(u)||0),m(t)&&(t=t.call(e,n,S.extend({},s))),null!=t.top&&(f.top=t.top-s.top+a),null!=t.left&&(f.left=t.left-s.left+i),"using"in t?t.using.call(e,f):("number"==typeof f.top&&(f.top+="px"),"number"==typeof f.left&&(f.left+="px"),c.css(f))}},S.fn.extend({offset:function(t){if(arguments.length)return void 0===t?this:this.each(function(e){S.offset.setOffset(this,t,e)});var e,n,r=this[0];return r?r.getClientRects().length?(e=r.getBoundingClientRect(),n=r.ownerDocument.defaultView,{top:e.top+n.pageYOffset,left:e.left+n.pageXOffset}):{top:0,left:0}:void 0},position:function(){if(this[0]){var e,t,n,r=this[0],i={top:0,left:0};if("fixed"===S.css(r,"position"))t=r.getBoundingClientRect();else{t=this.offset(),n=r.ownerDocument,e=r.offsetParent||n.documentElement;while(e&&(e===n.body||e===n.documentElement)&&"static"===S.css(e,"position"))e=e.parentNode;e&&e!==r&&1===e.nodeType&&((i=S(e).offset()).top+=S.css(e,"borderTopWidth",!0),i.left+=S.css(e,"borderLeftWidth",!0))}return{top:t.top-i.top-S.css(r,"marginTop",!0),left:t.left-i.left-S.css(r,"marginLeft",!0)}}},offsetParent:function(){return this.map(function(){var e=this.offsetParent;while(e&&"static"===S.css(e,"position"))e=e.offsetParent;return e||re})}}),S.each({scrollLeft:"pageXOffset",scrollTop:"pageYOffset"},function(t,i){var o="pageYOffset"===i;S.fn[t]=function(e){return $(this,function(e,t,n){var r;if(x(e)?r=e:9===e.nodeType&&(r=e.defaultView),void 0===n)return r?r[i]:e[t];r?r.scrollTo(o?r.pageXOffset:n,o?n:r.pageYOffset):e[t]=n},t,e,arguments.length)}}),S.each(["top","left"],function(e,n){S.cssHooks[n]=$e(y.pixelPosition,function(e,t){if(t)return t=Be(e,n),Me.test(t)?S(e).position()[n]+"px":t})}),S.each({Height:"height",Width:"width"},function(a,s){S.each({padding:"inner"+a,content:s,"":"outer"+a},function(r,o){S.fn[o]=function(e,t){var n=arguments.length&&(r||"boolean"!=typeof e),i=r||(!0===e||!0===t?"margin":"border");return $(this,function(e,t,n){var r;return x(e)?0===o.indexOf("outer")?e["inner"+a]:e.document.documentElement["client"+a]:9===e.nodeType?(r=e.documentElement,Math.max(e.body["scroll"+a],r["scroll"+a],e.body["offset"+a],r["offset"+a],r["client"+a])):void 0===n?S.css(e,t,i):S.style(e,t,n,i)},s,n?e:void 0,n)}})}),S.each(["ajaxStart","ajaxStop","ajaxComplete","ajaxError","ajaxSuccess","ajaxSend"],function(e,t){S.fn[t]=function(e){return this.on(t,e)}}),S.fn.extend({bind:function(e,t,n){return this.on(e,null,t,n)},unbind:function(e,t){return this.off(e,null,t)},delegate:function(e,t,n,r){return this.on(t,e,n,r)},undelegate:function(e,t,n){return 1===arguments.length?this.off(e,"**"):this.off(t,e||"**",n)},hover:function(e,t){return this.mouseenter(e).mouseleave(t||e)}}),S.each("blur focus focusin focusout resize scroll click dblclick mousedown mouseup mousemove mouseover mouseout mouseenter mouseleave change select submit keydown keypress keyup contextmenu".split(" "),function(e,n){S.fn[n]=function(e,t){return 0// class to convert the standard JSON representation of a gatenlp // document into something we need here and methods to access the data. var gatenlpDocRep = class { constructor(bdoc, parms) { this.sep = "║" this.sname2types = new Map(); this.snameid2ann = new Map(); this.snametype2ids = new Map(); this.text = bdoc["text"]; this.presel_list = parms["presel_list"] this.presel_set = new Set(parms["presel_set"]) this.cols4types = parms["cols4types"] if ("palette" in parms) { this.palette = parms["palette"] } const regex = / +$/; this.features = bdoc["features"]; if (this.text == null) { this.text = "[No proper GATENLP document to show]"; return; } let annsets = bdoc["annotation_sets"]; if (annsets == null) { return; } for (let setname in annsets) { // console.log("Processing setname: " + setname) let annset = annsets[setname]; let types4annset = new Set(); let anns4set = annset["annotations"]; for (let [idx, element] of anns4set.entries()) { // console.log("adding ann: " + idx + " / " + element) let annid = element["id"].toString(); let anntype = element["type"]; types4annset.add(anntype); // let snametype = setname + DocRep.sep + anntype; let snametype = setname + this.sep + anntype; // console.log("Created key " + snametype) let ids4type = this.snametype2ids.get(snametype); if (ids4type == null) { //console.log("Adding " + [annid]) this.snametype2ids.set(snametype, [annid]); // console.log("keys now " + Array.from(this.snametype2ids.keys())) } else { ids4type.push(annid); // console.log("snametype2ids for " + snametype + " is now " + ids4type) } let snameid = setname + this.sep + annid let ann4snameid = this.snameid2ann.get(snameid); if (ann4snameid == null) { this.snameid2ann.set(snameid, element); } else { // how to handle this odd error? } } this.sname2types.set(setname, Array.from(types4annset).sort()); } } // constructor

setnames() {
    return Array.from(this.sname2types.keys()).sort();
}

types4setname(setname) {
    // return a sorted list of annotation types for a set name
    return Array.from(this.sname2types.get(setname)); // already sorted!
}

annids4snametype(setname, anntype) {
    // return a list of annotation ids for a setname and annotation type
    return this.snametype2ids.get(setname + this.sep + anntype);
}

ann4setnameannid(setname, annid) {
    // return the annotation object (map) for a set/id
    return this.snameid2ann.get(setname + this.sep + annid)
}

anns4settype(setname, type) {
    //console.log("Getting anns for " + setname + " " + type)
    let annids = this.annids4snametype(setname, type);
    let anns = [];
    for (let annid of annids) {
        anns[anns.length] = this.ann4setnameannid(setname, annid);
    }
    //console.log("Found " + annids + " returning " + anns);
    return anns;
}

};

function docview_annsel(obj, ev, anns) { if (anns.size > 1) { // if there are several annotation, show the popup $(obj.id_popup).empty(); for (let info of anns.values()) { let fields = info.split("║") let setname = fields[0] let annid = fields[2]
let ann = obj.docrep.ann4setnameannid(setname, annid); // console.log("Looking up setname="+setname+",annid="+annid+" gave: "+ann) let feats = ann.features; let idpopup = obj.id_popup; $("

" + ann.type + ": id=" + annid + " offsets=" + ann.start + ".." + ann.end + " (" + (ann.end-ann.start) + ")" + "

").on("click", function(x) { docview_showAnn(obj, ann); $(idpopup).hide(); }).appendTo(obj.id_popup); } $(obj.id_popup).show();
} else if (anns.size == 1) { // if there is just one annotation, show features immediately, without the popup let a = anns.values().next()["value"] let fields = a.split("║")
let ann = obj.docrep.ann4setnameannid(fields[0], fields[2]); docview_showAnn(obj, ann); } else { console.error("EMPTY ANNS???"); } }

function docview_showFeatures(obj, features) { let tbl = $("

").attr("class", obj.idprefix+"featuretable"); for (let fname in features) { let fval = JSON.stringify(features[fname]); tbl.append(""); } $(obj.id_details).append(tbl); }

function docview_showAnn(obj, ann) { $(obj.id_details).empty(); $(obj.id_details).append("

Annotation: " + ann.type + ", id:" + ann.id + " offsets:" + ann.start + ".." + ann.end + " (" + (ann.end-ann.start) + ")

"); docview_showFeatures(obj, ann.features); }

function docview_showDocFeatures(obj, features) { $(obj.id_details).empty(); $(obj.id_details).append("

Document features:

"); docview_showFeatures(obj, features); }

function hex2rgba(hx) { return [ parseInt(hx.substring(1, 3), 16), parseInt(hx.substring(3, 5), 16), parseInt(hx.substring(5, 7), 16), 1.0 ]; };

// class to build the HTML for viewing the converted document var gatenlpDocView = class { constructor(docrep, idprefix="GATENLPID-", config=undefined) { // idprefix: the prefix to add to all ids and classes this.sep = "║" this.docrep = docrep; this.idprefix = idprefix; this.id_text = "#" + idprefix + "text"; this.id_chooser = "#" + idprefix + "chooser"; this.id_details = "#" + idprefix + "details"; this.id_popup = "#" + idprefix + "popup"; this.id_hdr = "#" + idprefix + "hdr"; this.id_dochdr = "#" + idprefix + "dochdr"; this.class_selection = idprefix + "selection"; this.class_fname = idprefix + "fname"; this.class_fvalue = idprefix + "fvalue"; this.class_label = idprefix + "label"; this.class_input = idprefix + "input"; this.chosen = []; this.anns4offset = undefined; // create default config here this.config = config; this.palettex = [ // modified from R lib pals: alphabet2 "#AA6DAA", "#3283FE", "#85660D", "#782AB6", "#565656", "#1C8356", "#16FF32", "#F7E1A0", "#E2E2E2", "#1CBE4F", "#C4451C", "#DEA0FD", "#FE00FA", "#325A9B", "#FEAF16", "#F8A19F", "#90AD1C", "#F6222E", "#1CFFCE", "#2ED9FF", "#B10DA1", "#C075A6", "#FC1CBF", "#B00068", "#FBE426", "#FA0087", // modified from R lib pals: polychrome "#5A5156", "#E4E1E3", "#F6222E", "#FE00FA", "#16FF32", "#3283FE", "#FEAF16", "#B00068", "#1CFFCE", "#90AD1C", "#2ED9FF", "#DEA0FD", "#AA0DFE", "#F8A19F", "#325A9B", "#C4451C", "#1C8356", "#85660D", "#B10DA1", "#FBE426", "#1CBE4F", "#FA0087", "#FC1CBF", "#F7E1A0", "#C075A6", "#782AB6", "#AAF400", "#BDCDFF", "#822E1C", "#B5EFB5", "#7ED7D1", "#1C7F93", "#D85FF7", "#683B79", "#66B0FF", "#3B00FB" ] if (typeof this.docrep.palette !== 'undefined') { this.palettex = this.docrep.palette } this.palette = this.palettex.map(hex2rgba) this.type2colour = new Map(); }

style4color(col) {
    return "background-color: rgba(" + col.join(",") + ");"
}

color4types(atypes) {
    // atypes is a list of set┼type┼annid strings
    let r = 0;
    let g = 0;
    let b = 0;
    let a = 0;
    for (let info of atypes.values()) {
        let fields = info.split(this.sep)
        let typ = fields[0] + this.sep + fields[1];
        let col = this.type2colour.get(typ);
        // console.log("Looked up color for "+typ+" got "+col)
        r += col[0];
        g += col[1];
        b += col[2];
        a += col[3];
    }
    r = Math.floor(r / atypes.size);
    g = Math.floor(g / atypes.size);
    b = Math.floor(b / atypes.size);
    a = a / atypes.size;
    // console.log("Final colors for len "+atypes.size+" r="+r+" g="+g)
    return [r, g, b, 1.0];
}

init() {
    let divcontent = $(this.id_text);
    $(divcontent).empty();
    let text = this.docrep.text;
    let thehtml = $.parseHTML(this.htmlEntities(text));
    $(divcontent).append(thehtml);

    // First of all, create the annotation chooser
    // create a form which contains:
    // for each annotation set create an a tag. followed by a div that contains all the checkbox fields
    let divchooser = $(this.id_chooser);
    $(divchooser).empty();
    let formchooser = $("<form>");
    let colidx = 0
    for (let setname of this.docrep.setnames()) {
        let setname2show = setname;
        // TODO: add number of annotations in the set in parentheses
        if (setname == "") {
            setname2show = "[Default Set]"
        }
        // TODO: make what we show here configurable?
        $(formchooser).append($(document.createElement('div')).attr("class", this.id_hdr).append(setname2show))
        let div4set = document.createElement("div")
        // $(div4set).attr("id", setname);
        $(div4set).attr("style", "margin-bottom: 10px;");
        for (let anntype of this.docrep.types4setname(setname)) {
            //console.log("Addingsss type " + anntype)
            let setandtype = setname + this.docrep.sep + anntype;
            let col = undefined
            if (setandtype in this.docrep.cols4types) {
                col = hex2rgba(this.docrep.cols4types[setandtype])
            } else {
                col = this.palette[colidx];
            }
            this.type2colour.set(setname + this.sep + anntype, col);
            colidx = (colidx + 1) % this.palette.length;
            let lbl = $("<label>").attr({ "style": this.style4color(col), "class": this.class_label });
            let object = this
            let annhandler = function(ev) { docview_annchosen(object, ev, setname, anntype) }
            let inp = $('<input type="checkbox">').attr({ "type": "checkbox", "class": this.class_input, "data-anntype": anntype, "data-setname": setname}).on("click", annhandler)
            if (this.docrep.presel_set.has(setandtype)) {
                inp.attr("checked", "")
            }
            $(lbl).append(inp);
            $(lbl).append(anntype);
            // append the number of annotations in this set 
            let n = this.docrep.annids4snametype(setname, anntype).length;
            $(lbl).append(" (" + n + ")");
            $(div4set).append(lbl)
            $(div4set).append($("<br>"))
            $(divchooser).append(formchooser)
        }
        $(formchooser).append(div4set)
    }

    let obj = this;
    let feats = this.docrep["features"];
    docview_showDocFeatures(obj, feats);
    $(this.id_dochdr).text("Document:").on("click", function(ev) { docview_showDocFeatures(obj, feats) });
    this.chosen = this.docrep.presel_list
    this.buildAnns4Offset()
    this.buildContent()
}

    set2list(theset) {
        let arr = new Array()
        for (var el of theset.values()) {
           arr[arr.length] = el
        }
        return arr
    }

    setsequal(set1, set2) {
        if (set1.size !== set2.size) return false;
        for (var el of set1) if (!set2.has(el)) return false;
        return true;
    }

buildAnns4Offset() {
    // console.log("Running buildAnns4Offset")
    //this.anns4offset = new Array(this.docrep.text.length + 1);
    this.anns4offset = new Array()

    // for all the set/type combinations that have been selected ... 
    for (let [sname, atype] of this.chosen) {
        //console.log("sname/type: " + sname + "/" + atype);
        // get the list of annotations that match the given Setname and annotation type
        let anns = this.docrep.anns4settype(sname, atype);
        for (let ann of anns) {
            // console.log("processing ann: " + ann + " start=" + ann.start + " end=" + ann.end + " type=" + ann.type)
            // store the annotation setname/typename/annid for each offset of each annotation
            // to indicate the end of the annotation also store an empty list for the offset after the annotation 
            // unless we already have something there

            // trick for zero length annotations: show them as length one annotations for now
            var endoff = ann.end
            if (ann.start == ann.end) endoff = endoff+1
            for (let i = ann.start; i < endoff; i++) { // iterate until one beyond the end of the ann
                let have = this.anns4offset[i]
                if (have == undefined) {                    
                  have = { "offset": i, "anns": new Set()}
                  this.anns4offset[i] = have
                }
                if (i < endoff) {
                    // append a new set/type tuple to the list of set/types at this offset
                    let tmp = this.anns4offset[i]["anns"];
                    let toadd = sname + this.sep + atype + this.sep + ann.id
                    // console.log("Trying to add "+toadd+" to "+this.set2list(tmp))
                    tmp = tmp.add(toadd); 
                    //console.log("is now "+this.set2list(tmp))
                    //console.log("entry for offset "+i+" is now " + this.set2list(this.anns4offset[i]["anns"]));
                }
            }
        }
    }
    //console.log("initial anns4Offset:")
    //console.log(this.anns4offset)
    // now all offsets have a list of set/type and set/annid tuples
    // compress the list to only contain anything but undefined where it changes 
    let last = this.anns4offset[0]
    for (let i = 1; i < this.anns4offset.length+1; i++) {
        let cur = this.anns4offset[i]
        if (last == undefined && cur == undefined) {
            // console.log("Offset "+i+" both undefined")
            // nothing to do
        } else if (last == undefined && cur != undefined) {
            // we have a new list of annotations, keep it: nothing to do
            //console.log("Offset "+i+" last undefined, this one not")
        } else if (last != undefined && cur == undefined) {
            // we switch from some list of annotations to the empty list: 
            // add an empty entry
            //console.log("Offset "+i+" last one not undefined, this undefined, inserting empty list")
            this.anns4offset[i] = { "anns": new Set(), "offset": i}
        } else {
            // both offsets have annotations, but do the differ? we need to compare the types and annids
            // For now we do this by comparing the stringified representations
            let s1 = last["anns"]
            let s2 = cur["anns"]
            // console.log("Offset "+i+" Cur: "+this.set2list(s2)+" last: "+this.set2list(s1))
            if (this.setsequal(s1,s2)) {
               // console.log("Detected equal")
               this.anns4offset[i] = undefined
            }
        } 
        last = cur
    }
let beyond = this.docrep.text.length
this.anns4offset[beyond] = { "anns": new Set(), "offset": beyond}

    // console.log("compressed anns4Offset:")
    // console.log(this.anns4offset)

}

buildContent() {
    //console.log("Running buildContent");
    // got through all the offsets and check where the annotations change
    // start with the set of annotations in the first offset (empty if undefined) as lastset, calculate color for set
    // go through all subsequent offsets
    // when we find an entry where the annotations change:
    // * get the annotation setname/types 
    // * from the list of setname/types, determine a colour and store it
    // * generate the span from last to here 
    // * process one additional char at the end to include last span
    let spans = []
    let last = this.anns4offset[0];
    if (last == undefined) {
        last = { "anns": new Set(), "offset": 0 };
    }
    for (let i = 1; i < this.anns4offset.length+1; i++) {
        let info = this.anns4offset[i];
        if (info != undefined) {
            let txt = this.docrep.text.substring(last["offset"], info["offset"]);
            txt = txt.replace(/\n/g, "\u2002\n");
            // console.log("Got text: "+txt) 
            let span = undefined;
            if (last["anns"].size != 0) {
                let col = this.color4types(last.anns);
                let sty = this.style4color(col)+"white-space:pre-wrap;" 
                span = $('<span>').attr("style", sty);
                let object = this;
                let anns = last.anns;
                let annhandler = function(ev) { docview_annsel(object, ev, anns) }
                span.on("click", annhandler);
                // console.log("Adding styled text for "+col+"/"+sty+" : "+txt)                    
            } else {
                // console.log("Adding non-styled text "+txt)
                span = $('<span>');
            }
            span.append($.parseHTML(this.htmlEntities(txt)));
            spans.push(span);
            last = info;
        }
    }
    // Replace the content
    let divcontent = $(this.id_text);
    $(divcontent).empty();
    $(divcontent).append(spans);
}

htmlEntities(str) {
    return str.replaceAll('&', '&amp;').replaceAll('<', '&lt;').replaceAll('>', '&gt;').replaceAll('"', '&quot;').replaceAll("\n", '<br>');
}

}; </script>

.YEJDYGXGFY-row { width: 100%; display: flex; flex-direction: row; flex-wrap: nowrap; }

.YEJDYGXGFY-col { border: 1px solid grey; display: inline-block; min-width: 200px; padding: 5px; /* white-space: normal; / / white-space: pre-wrap; */ overflow-y: auto; }

.YEJDYGXGFY-hdr { font-size: 1.2rem; font-weight: bold; }

.YEJDYGXGFY-label { margin-bottom: -15px; display: block; }

.YEJDYGXGFY-input { vertical-align: middle; position: relative; *overflow: hidden; }

#YEJDYGXGFY-popup { display: none; color: black; position: absolute; margin-top: 10%; margin-left: 10%; background: #aaaaaa; width: 60%; height: 60%; z-index: 50; padding: 25px 25px 25px; border: 1px solid black; overflow: auto; }

.YEJDYGXGFY-selection { margin-bottom: 5px; }

.YEJDYGXGFY-featuretable { margin-top: 10px; }

.YEJDYGXGFY-fname { text-align: left !important; font-weight: bold; margin-right: 10px; } .YEJDYGXGFY-fvalue { text-align: left !important; } </style>

<script type="text/javascript">
let YEJDYGXGFY_data = {"annotation_sets": {"": {"name": "detached-from:", "annotations": [{"type": "Date", "start": 38, "end": 48, "id": 0, "features": {}}, {"type": "Date", "start": 64, "end": 74, "id": 1, "features": {}}], "next_annid": 2}}, "text": "A document that contains a date here: 2013-01-12 and also here: 1999-12-31", "features": {}, "offset_type": "j", "name": ""} ; 
let YEJDYGXGFY_parms = {"presel_set": [], "presel_list": [], "cols4types": {}} ;
new gatenlpDocView(new gatenlpDocRep(YEJDYGXGFY_data, YEJDYGXGFY_parms), "YEJDYGXGFY-").init();
</script>

The rules file/string format

A rules file must contain one or more rules.

Each rule consists of:

one or more pattern lines which must start with "|", followed by
one or more action lines which must start with a comma separated list of group numbers followed by "=>" followed by the annotation type to assign, optionally followed by feature assignments.

The action line specifies how an annotation should get created for one or more groups of a matching regular expression.

The simple rules string above contains one rule, with one patterh line and one action line:

|[0-9]{4}-[0-9]{2}-[0-9]{2}
0 => Date

The pattern line |[0-9]{4}-[0-9]{2}-[0-9]{2} specifies the simple regular expression.

The action line 0 => Date specifies that an annotation with the annotation type "Date" should get created for the match, spanning "group 0". The convention with regular expressions is that "group 0" always referes to whatever is matched by the whole regular expression.

Using groups

In addition to group 0, anything within simple parentheses in the regular expression is a "capturing group". Capturing groups get numberd by their opening parenthesis when counting from left to right. For example, the following regular expression has 3 additional groups for the year, month and day part of the whole ISO date. The rule then refers to the whole matched date via group 0 but also creates annotations of type Year, Month and Day for each of the groups:

|([0-9]{4})-([0-9]{2})-([0-9]{2})
0 => Date
1 => Year
2 => Month
3 => Day

rules2 = """
|([0-9]{4})-([0-9]{2})-([0-9]{2})
0 => Date
1 => Year
2 => Month
3 => Day
"""

annt2 = StringRegexAnnotator(source=rules2, source_fmt="string")

doc2 = Document("A document that contains a date here: 2013-01-12 and also here: 1999-12-31")

annt2(doc2)
doc2

.SCZPNLIREU-row { width: 100%; display: flex; flex-direction: row; flex-wrap: nowrap; }

.SCZPNLIREU-col { border: 1px solid grey; display: inline-block; min-width: 200px; padding: 5px; /* white-space: normal; / / white-space: pre-wrap; */ overflow-y: auto; }

.SCZPNLIREU-hdr { font-size: 1.2rem; font-weight: bold; }

.SCZPNLIREU-label { margin-bottom: -15px; display: block; }

.SCZPNLIREU-input { vertical-align: middle; position: relative; *overflow: hidden; }

#SCZPNLIREU-popup { display: none; color: black; position: absolute; margin-top: 10%; margin-left: 10%; background: #aaaaaa; width: 60%; height: 60%; z-index: 50; padding: 25px 25px 25px; border: 1px solid black; overflow: auto; }

.SCZPNLIREU-selection { margin-bottom: 5px; }

.SCZPNLIREU-featuretable { margin-top: 10px; }

.SCZPNLIREU-fname { text-align: left !important; font-weight: bold; margin-right: 10px; } .SCZPNLIREU-fvalue { text-align: left !important; } </style>

<script type="text/javascript">
let SCZPNLIREU_data = {"annotation_sets": {"": {"name": "detached-from:", "annotations": [{"type": "Date", "start": 38, "end": 48, "id": 0, "features": {}}, {"type": "Year", "start": 38, "end": 42, "id": 1, "features": {}}, {"type": "Month", "start": 43, "end": 45, "id": 2, "features": {}}, {"type": "Day", "start": 46, "end": 48, "id": 3, "features": {}}, {"type": "Date", "start": 64, "end": 74, "id": 4, "features": {}}, {"type": "Year", "start": 64, "end": 68, "id": 5, "features": {}}, {"type": "Month", "start": 69, "end": 71, "id": 6, "features": {}}, {"type": "Day", "start": 72, "end": 74, "id": 7, "features": {}}], "next_annid": 8}}, "text": "A document that contains a date here: 2013-01-12 and also here: 1999-12-31", "features": {}, "offset_type": "j", "name": ""} ; 
let SCZPNLIREU_parms = {"presel_set": [], "presel_list": [], "cols4types": {}} ;
new gatenlpDocView(new gatenlpDocRep(SCZPNLIREU_data, SCZPNLIREU_parms), "SCZPNLIREU-").init();
</script>

Adding features to annotations

For each annotation that gets created for a match it is possible to also specify features to set in each action. Feature values can be specified as constants or as the value of one of the matched groups. To illustrate this, the following example assigns the year, month and day string to all annotations (Date, Day, Month, Year). In addition it assigns the constant value "iso" to the "type" feature of the "Date" annotation. To assign the value of some group number n, the variable "Gn" can be used, e.g. "G2" for group 2:

rules3 = """
|([0-9]{4})-([0-9]{2})-([0-9]{2})
0 => Date  type="iso", year=G1, month=G2, day=G3
1 => Year  year=G1, month=G2, day=G3
2 => Month year=G1, month=G2, day=G3
3 => Day year=G1, month=G2, day=G3
"""

annt3 = StringRegexAnnotator(source=rules3, source_fmt="string")

doc3 = Document("A document that contains a date here: 2013-01-12 and also here: 1999-12-31")

annt3(doc3)
doc3

.FGSTASDZPG-row { width: 100%; display: flex; flex-direction: row; flex-wrap: nowrap; }

.FGSTASDZPG-col { border: 1px solid grey; display: inline-block; min-width: 200px; padding: 5px; /* white-space: normal; / / white-space: pre-wrap; */ overflow-y: auto; }

.FGSTASDZPG-hdr { font-size: 1.2rem; font-weight: bold; }

.FGSTASDZPG-label { margin-bottom: -15px; display: block; }

.FGSTASDZPG-input { vertical-align: middle; position: relative; *overflow: hidden; }

#FGSTASDZPG-popup { display: none; color: black; position: absolute; margin-top: 10%; margin-left: 10%; background: #aaaaaa; width: 60%; height: 60%; z-index: 50; padding: 25px 25px 25px; border: 1px solid black; overflow: auto; }

.FGSTASDZPG-selection { margin-bottom: 5px; }

.FGSTASDZPG-featuretable { margin-top: 10px; }

.FGSTASDZPG-fname { text-align: left !important; font-weight: bold; margin-right: 10px; } .FGSTASDZPG-fvalue { text-align: left !important; } </style>

<script type="text/javascript">
let FGSTASDZPG_data = {"annotation_sets": {"": {"name": "detached-from:", "annotations": [{"type": "Date", "start": 38, "end": 48, "id": 0, "features": {"type": "iso", "year": "2013", "month": "01", "day": "12"}}, {"type": "Year", "start": 38, "end": 42, "id": 1, "features": {"year": "2013", "month": "01", "day": "12"}}, {"type": "Month", "start": 43, "end": 45, "id": 2, "features": {"year": "2013", "month": "01", "day": "12"}}, {"type": "Day", "start": 46, "end": 48, "id": 3, "features": {"year": "2013", "month": "01", "day": "12"}}, {"type": "Date", "start": 64, "end": 74, "id": 4, "features": {"type": "iso", "year": "1999", "month": "12", "day": "31"}}, {"type": "Year", "start": 64, "end": 68, "id": 5, "features": {"year": "1999", "month": "12", "day": "31"}}, {"type": "Month", "start": 69, "end": 71, "id": 6, "features": {"year": "1999", "month": "12", "day": "31"}}, {"type": "Day", "start": 72, "end": 74, "id": 7, "features": {"year": "1999", "month": "12", "day": "31"}}], "next_annid": 8}}, "text": "A document that contains a date here: 2013-01-12 and also here: 1999-12-31", "features": {}, "offset_type": "j", "name": ""} ; 
let FGSTASDZPG_parms = {"presel_set": [], "presel_list": [], "cols4types": {}} ;
new gatenlpDocView(new gatenlpDocRep(FGSTASDZPG_data, FGSTASDZPG_parms), "FGSTASDZPG-").init();
</script>

Having more than one rule

A rule file/string can contain any number of rules. The following example includes 2 rules, matching either an ISO date, or a traditional date (DD/MM/YYYY). The example also contains comment lines, which start either with a "#" or a double slash "//":

rules4 = """
// The ISO date:
|([0-9]{4})-([0-9]{2})-([0-9]{2})
0 => Date  type="iso", year=G1, month=G2, day=G3

# The traditional way of writing a date:
|([0-9]{2})/([0-9]{2})/([0-9]{4})
0 => Date  type="traditional", year=G3, month=G2, day=G1
"""

annt4 = StringRegexAnnotator(source=rules4, source_fmt="string")

doc4 = Document("A document that contains a date here: 2013-01-12 and also here: 14/02/1991")

annt4(doc4)
doc4

.NEQKMCRSHA-row { width: 100%; display: flex; flex-direction: row; flex-wrap: nowrap; }

.NEQKMCRSHA-col { border: 1px solid grey; display: inline-block; min-width: 200px; padding: 5px; /* white-space: normal; / / white-space: pre-wrap; */ overflow-y: auto; }

.NEQKMCRSHA-hdr { font-size: 1.2rem; font-weight: bold; }

.NEQKMCRSHA-label { margin-bottom: -15px; display: block; }

.NEQKMCRSHA-input { vertical-align: middle; position: relative; *overflow: hidden; }

#NEQKMCRSHA-popup { display: none; color: black; position: absolute; margin-top: 10%; margin-left: 10%; background: #aaaaaa; width: 60%; height: 60%; z-index: 50; padding: 25px 25px 25px; border: 1px solid black; overflow: auto; }

.NEQKMCRSHA-selection { margin-bottom: 5px; }

.NEQKMCRSHA-featuretable { margin-top: 10px; }

.NEQKMCRSHA-fname { text-align: left !important; font-weight: bold; margin-right: 10px; } .NEQKMCRSHA-fvalue { text-align: left !important; } </style>

<script type="text/javascript">
let NEQKMCRSHA_data = {"annotation_sets": {"": {"name": "detached-from:", "annotations": [{"type": "Date", "start": 38, "end": 48, "id": 0, "features": {"type": "iso", "year": "2013", "month": "01", "day": "12"}}, {"type": "Date", "start": 64, "end": 74, "id": 1, "features": {"type": "traditional", "year": "1991", "month": "02", "day": "14"}}], "next_annid": 2}}, "text": "A document that contains a date here: 2013-01-12 and also here: 14/02/1991", "features": {}, "offset_type": "j", "name": ""} ; 
let NEQKMCRSHA_parms = {"presel_set": [], "presel_list": [], "cols4types": {}} ;
new gatenlpDocView(new gatenlpDocRep(NEQKMCRSHA_data, NEQKMCRSHA_parms), "NEQKMCRSHA-").init();
</script>

Longest match only

Dates are sometimes written with 2 digits for the year only. The following example has two rules for a traditional date format. Because of the parameter longest_only=False the second date now matches both the first and the second rule.

rules5 = """
# The traditional way of writing a date, 2 digit year
|([0-9]{2})/([0-9]{2})/([0-9]{2})
0 => Date  type="traditional-short", year=G3, month=G2, day=G1

# The traditional way of writing a date, 4 digit year
|([0-9]{2})/([0-9]{2})/([0-9]{4})
0 => Date  type="traditional-long", year=G3, month=G2, day=G1

"""

annt5 = StringRegexAnnotator(source=rules5, source_fmt="string", longest_only=False)

doc5 = Document("A document that contains a date here: 12/04/98 and also here: 14/02/1991")

annt5(doc5)
doc5

.JQEHAFQXAW-row { width: 100%; display: flex; flex-direction: row; flex-wrap: nowrap; }

.JQEHAFQXAW-col { border: 1px solid grey; display: inline-block; min-width: 200px; padding: 5px; /* white-space: normal; / / white-space: pre-wrap; */ overflow-y: auto; }

.JQEHAFQXAW-hdr { font-size: 1.2rem; font-weight: bold; }

.JQEHAFQXAW-label { margin-bottom: -15px; display: block; }

.JQEHAFQXAW-input { vertical-align: middle; position: relative; *overflow: hidden; }

#JQEHAFQXAW-popup { display: none; color: black; position: absolute; margin-top: 10%; margin-left: 10%; background: #aaaaaa; width: 60%; height: 60%; z-index: 50; padding: 25px 25px 25px; border: 1px solid black; overflow: auto; }

.JQEHAFQXAW-selection { margin-bottom: 5px; }

.JQEHAFQXAW-featuretable { margin-top: 10px; }

.JQEHAFQXAW-fname { text-align: left !important; font-weight: bold; margin-right: 10px; } .JQEHAFQXAW-fvalue { text-align: left !important; } </style>

<script type="text/javascript">
let JQEHAFQXAW_data = {"annotation_sets": {"": {"name": "detached-from:", "annotations": [{"type": "Date", "start": 38, "end": 46, "id": 0, "features": {"type": "traditional-short", "year": "98", "month": "04", "day": "12"}}, {"type": "Date", "start": 62, "end": 70, "id": 1, "features": {"type": "traditional-short", "year": "19", "month": "02", "day": "14"}}, {"type": "Date", "start": 62, "end": 72, "id": 2, "features": {"type": "traditional-long", "year": "1991", "month": "02", "day": "14"}}], "next_annid": 3}}, "text": "A document that contains a date here: 12/04/98 and also here: 14/02/1991", "features": {}, "offset_type": "j", "name": ""} ; 
let JQEHAFQXAW_parms = {"presel_set": [], "presel_list": [], "cols4types": {}} ;
new gatenlpDocView(new gatenlpDocRep(JQEHAFQXAW_data, JQEHAFQXAW_parms), "JQEHAFQXAW-").init();
</script>

With longest_only=True at each matching position, only the longest match (or longest matches if there are several matches with the same longest length) are annotated. Now only the rule that produces the longer match is used:

annt5a = StringRegexAnnotator(source=rules5, source_fmt="string", longest_only=True)

doc5a = Document("A document that contains a date here: 12/04/98 and also here: 14/02/1991")

annt5a(doc5a)
doc5a

.OSRPKKHSWF-row { width: 100%; display: flex; flex-direction: row; flex-wrap: nowrap; }

.OSRPKKHSWF-col { border: 1px solid grey; display: inline-block; min-width: 200px; padding: 5px; /* white-space: normal; / / white-space: pre-wrap; */ overflow-y: auto; }

.OSRPKKHSWF-hdr { font-size: 1.2rem; font-weight: bold; }

.OSRPKKHSWF-label { margin-bottom: -15px; display: block; }

.OSRPKKHSWF-input { vertical-align: middle; position: relative; *overflow: hidden; }

#OSRPKKHSWF-popup { display: none; color: black; position: absolute; margin-top: 10%; margin-left: 10%; background: #aaaaaa; width: 60%; height: 60%; z-index: 50; padding: 25px 25px 25px; border: 1px solid black; overflow: auto; }

.OSRPKKHSWF-selection { margin-bottom: 5px; }

.OSRPKKHSWF-featuretable { margin-top: 10px; }

.OSRPKKHSWF-fname { text-align: left !important; font-weight: bold; margin-right: 10px; } .OSRPKKHSWF-fvalue { text-align: left !important; } </style>

<script type="text/javascript">
let OSRPKKHSWF_data = {"annotation_sets": {"": {"name": "detached-from:", "annotations": [{"type": "Date", "start": 38, "end": 46, "id": 0, "features": {"type": "traditional-short", "year": "98", "month": "04", "day": "12"}}, {"type": "Date", "start": 62, "end": 72, "id": 1, "features": {"type": "traditional-long", "year": "1991", "month": "02", "day": "14"}}], "next_annid": 2}}, "text": "A document that contains a date here: 12/04/98 and also here: 14/02/1991", "features": {}, "offset_type": "j", "name": ""} ; 
let OSRPKKHSWF_parms = {"presel_set": [], "presel_list": [], "cols4types": {}} ;
new gatenlpDocView(new gatenlpDocRep(OSRPKKHSWF_data, OSRPKKHSWF_parms), "OSRPKKHSWF-").init();
</script>

Choosing matching rules

It is possible that several rules match the same position. The select_rules parameter can be used to configure which of all matching rules should actually be used. The default is "all", so all matching rules are considered, but if longest_only=True then only the longest of all rules are considered.

If select_rules="first" then whichever rule is the first (in order of appearance in the rule file/string) to match is the one used, all other rules which may also match at a position are ignored. Similarly, if select_rules="last" only the last of all matching rules is used.

In the following example, longest_only=False and select_rules="first" so the first rule that matches is the only one used:

annt5b = StringRegexAnnotator(source=rules5, source_fmt="string", longest_only=False, select_rules="first")

doc5b = Document("A document that contains a date here: 12/04/98 and also here: 14/02/1991")

annt5b(doc5b)
doc5b

.RYAFTGJDWG-row { width: 100%; display: flex; flex-direction: row; flex-wrap: nowrap; }

.RYAFTGJDWG-col { border: 1px solid grey; display: inline-block; min-width: 200px; padding: 5px; /* white-space: normal; / / white-space: pre-wrap; */ overflow-y: auto; }

.RYAFTGJDWG-hdr { font-size: 1.2rem; font-weight: bold; }

.RYAFTGJDWG-label { margin-bottom: -15px; display: block; }

.RYAFTGJDWG-input { vertical-align: middle; position: relative; *overflow: hidden; }

#RYAFTGJDWG-popup { display: none; color: black; position: absolute; margin-top: 10%; margin-left: 10%; background: #aaaaaa; width: 60%; height: 60%; z-index: 50; padding: 25px 25px 25px; border: 1px solid black; overflow: auto; }

.RYAFTGJDWG-selection { margin-bottom: 5px; }

.RYAFTGJDWG-featuretable { margin-top: 10px; }

.RYAFTGJDWG-fname { text-align: left !important; font-weight: bold; margin-right: 10px; } .RYAFTGJDWG-fvalue { text-align: left !important; } </style>

<script type="text/javascript">
let RYAFTGJDWG_data = {"annotation_sets": {"": {"name": "detached-from:", "annotations": [{"type": "Date", "start": 38, "end": 46, "id": 0, "features": {"type": "traditional-short", "year": "98", "month": "04", "day": "12"}}, {"type": "Date", "start": 62, "end": 70, "id": 1, "features": {"type": "traditional-short", "year": "19", "month": "02", "day": "14"}}], "next_annid": 2}}, "text": "A document that contains a date here: 12/04/98 and also here: 14/02/1991", "features": {}, "offset_type": "j", "name": ""} ; 
let RYAFTGJDWG_parms = {"presel_set": [], "presel_list": [], "cols4types": {}} ;
new gatenlpDocView(new gatenlpDocRep(RYAFTGJDWG_data, RYAFTGJDWG_parms), "RYAFTGJDWG-").init();
</script>

Overlapping matches

Sometimes it is possible that matches from different rules or the same rule can overlap, here is a simple example: the following rule simply matches any number of basic ASCII lower case characters. At each position where such a sequence starts, a match is found and an annotation is created.

rules6a = """
|[a-z]+
0 => Match
"""

annt6a = StringRegexAnnotator(source=rules6a, source_fmt="string")

doc6a = Document("A document that contains a date here: 12/04/98 and also here: 14/02/1991")

annt6a(doc6a)
print("Matching:", [doc6a[a] for a in doc6a.annset()])
doc6a

Matching: ['document', 'ocument', 'cument', 'ument', 'ment', 'ent', 'nt', 't', 'that', 'hat', 'at', 't', 'contains', 'ontains', 'ntains', 'tains', 'ains', 'ins', 'ns', 's', 'a', 'date', 'ate', 'te', 'e', 'here', 'ere', 're', 'e', 'and', 'nd', 'd', 'also', 'lso', 'so', 'o', 'here', 'ere', 're', 'e']

.GWBEYZYFPY-row { width: 100%; display: flex; flex-direction: row; flex-wrap: nowrap; }

.GWBEYZYFPY-col { border: 1px solid grey; display: inline-block; min-width: 200px; padding: 5px; /* white-space: normal; / / white-space: pre-wrap; */ overflow-y: auto; }

.GWBEYZYFPY-hdr { font-size: 1.2rem; font-weight: bold; }

.GWBEYZYFPY-label { margin-bottom: -15px; display: block; }

.GWBEYZYFPY-input { vertical-align: middle; position: relative; *overflow: hidden; }

#GWBEYZYFPY-popup { display: none; color: black; position: absolute; margin-top: 10%; margin-left: 10%; background: #aaaaaa; width: 60%; height: 60%; z-index: 50; padding: 25px 25px 25px; border: 1px solid black; overflow: auto; }

.GWBEYZYFPY-selection { margin-bottom: 5px; }

.GWBEYZYFPY-featuretable { margin-top: 10px; }

.GWBEYZYFPY-fname { text-align: left !important; font-weight: bold; margin-right: 10px; } .GWBEYZYFPY-fvalue { text-align: left !important; } </style>

<script type="text/javascript">
let GWBEYZYFPY_data = {"annotation_sets": {"": {"name": "detached-from:", "annotations": [{"type": "Match", "start": 2, "end": 10, "id": 0, "features": {}}, {"type": "Match", "start": 3, "end": 10, "id": 1, "features": {}}, {"type": "Match", "start": 4, "end": 10, "id": 2, "features": {}}, {"type": "Match", "start": 5, "end": 10, "id": 3, "features": {}}, {"type": "Match", "start": 6, "end": 10, "id": 4, "features": {}}, {"type": "Match", "start": 7, "end": 10, "id": 5, "features": {}}, {"type": "Match", "start": 8, "end": 10, "id": 6, "features": {}}, {"type": "Match", "start": 9, "end": 10, "id": 7, "features": {}}, {"type": "Match", "start": 11, "end": 15, "id": 8, "features": {}}, {"type": "Match", "start": 12, "end": 15, "id": 9, "features": {}}, {"type": "Match", "start": 13, "end": 15, "id": 10, "features": {}}, {"type": "Match", "start": 14, "end": 15, "id": 11, "features": {}}, {"type": "Match", "start": 16, "end": 24, "id": 12, "features": {}}, {"type": "Match", "start": 17, "end": 24, "id": 13, "features": {}}, {"type": "Match", "start": 18, "end": 24, "id": 14, "features": {}}, {"type": "Match", "start": 19, "end": 24, "id": 15, "features": {}}, {"type": "Match", "start": 20, "end": 24, "id": 16, "features": {}}, {"type": "Match", "start": 21, "end": 24, "id": 17, "features": {}}, {"type": "Match", "start": 22, "end": 24, "id": 18, "features": {}}, {"type": "Match", "start": 23, "end": 24, "id": 19, "features": {}}, {"type": "Match", "start": 25, "end": 26, "id": 20, "features": {}}, {"type": "Match", "start": 27, "end": 31, "id": 21, "features": {}}, {"type": "Match", "start": 28, "end": 31, "id": 22, "features": {}}, {"type": "Match", "start": 29, "end": 31, "id": 23, "features": {}}, {"type": "Match", "start": 30, "end": 31, "id": 24, "features": {}}, {"type": "Match", "start": 32, "end": 36, "id": 25, "features": {}}, {"type": "Match", "start": 33, "end": 36, "id": 26, "features": {}}, {"type": "Match", "start": 34, "end": 36, "id": 27, "features": {}}, {"type": "Match", "start": 35, "end": 36, "id": 28, "features": {}}, {"type": "Match", "start": 47, "end": 50, "id": 29, "features": {}}, {"type": "Match", "start": 48, "end": 50, "id": 30, "features": {}}, {"type": "Match", "start": 49, "end": 50, "id": 31, "features": {}}, {"type": "Match", "start": 51, "end": 55, "id": 32, "features": {}}, {"type": "Match", "start": 52, "end": 55, "id": 33, "features": {}}, {"type": "Match", "start": 53, "end": 55, "id": 34, "features": {}}, {"type": "Match", "start": 54, "end": 55, "id": 35, "features": {}}, {"type": "Match", "start": 56, "end": 60, "id": 36, "features": {}}, {"type": "Match", "start": 57, "end": 60, "id": 37, "features": {}}, {"type": "Match", "start": 58, "end": 60, "id": 38, "features": {}}, {"type": "Match", "start": 59, "end": 60, "id": 39, "features": {}}], "next_annid": 40}}, "text": "A document that contains a date here: 12/04/98 and also here: 14/02/1991", "features": {}, "offset_type": "j", "name": ""} ; 
let GWBEYZYFPY_parms = {"presel_set": [], "presel_list": [], "cols4types": {}} ;
new gatenlpDocView(new gatenlpDocRep(GWBEYZYFPY_data, GWBEYZYFPY_parms), "GWBEYZYFPY-").init();
</script>

In such cases, it is often desirable to only try and find a match after any match that has already been found, so in this case, once "document" has been matched, only try and find the next match after the end of that match. This can be achieved by setting the parameter skip_longest=True:

rules6b = """
|[a-z]+
0 => Match
"""

annt6b = StringRegexAnnotator(source=rules6b, source_fmt="string", skip_longest=True)

doc6b = Document("A document that contains a date here: 12/04/98 and also here: 14/02/1991")

annt6b(doc6b)
print("Matching:", [doc6b[a] for a in doc6b.annset()])
doc6b

Matching: ['document', 'that', 'contains', 'a', 'date', 'here', 'and', 'also', 'here']

.CQYQCBKPGQ-row { width: 100%; display: flex; flex-direction: row; flex-wrap: nowrap; }

.CQYQCBKPGQ-col { border: 1px solid grey; display: inline-block; min-width: 200px; padding: 5px; /* white-space: normal; / / white-space: pre-wrap; */ overflow-y: auto; }

.CQYQCBKPGQ-hdr { font-size: 1.2rem; font-weight: bold; }

.CQYQCBKPGQ-label { margin-bottom: -15px; display: block; }

.CQYQCBKPGQ-input { vertical-align: middle; position: relative; *overflow: hidden; }

#CQYQCBKPGQ-popup { display: none; color: black; position: absolute; margin-top: 10%; margin-left: 10%; background: #aaaaaa; width: 60%; height: 60%; z-index: 50; padding: 25px 25px 25px; border: 1px solid black; overflow: auto; }

.CQYQCBKPGQ-selection { margin-bottom: 5px; }

.CQYQCBKPGQ-featuretable { margin-top: 10px; }

.CQYQCBKPGQ-fname { text-align: left !important; font-weight: bold; margin-right: 10px; } .CQYQCBKPGQ-fvalue { text-align: left !important; } </style>

<script type="text/javascript">
let CQYQCBKPGQ_data = {"annotation_sets": {"": {"name": "detached-from:", "annotations": [{"type": "Match", "start": 2, "end": 10, "id": 0, "features": {}}, {"type": "Match", "start": 11, "end": 15, "id": 1, "features": {}}, {"type": "Match", "start": 16, "end": 24, "id": 2, "features": {}}, {"type": "Match", "start": 25, "end": 26, "id": 3, "features": {}}, {"type": "Match", "start": 27, "end": 31, "id": 4, "features": {}}, {"type": "Match", "start": 32, "end": 36, "id": 5, "features": {}}, {"type": "Match", "start": 47, "end": 50, "id": 6, "features": {}}, {"type": "Match", "start": 51, "end": 55, "id": 7, "features": {}}, {"type": "Match", "start": 56, "end": 60, "id": 8, "features": {}}], "next_annid": 9}}, "text": "A document that contains a date here: 12/04/98 and also here: 14/02/1991", "features": {}, "offset_type": "j", "name": ""} ; 
let CQYQCBKPGQ_parms = {"presel_set": [], "presel_list": [], "cols4types": {}} ;
new gatenlpDocView(new gatenlpDocRep(CQYQCBKPGQ_data, CQYQCBKPGQ_parms), "CQYQCBKPGQ-").init();
</script>

Macros: creating complex regular expressions from simpler ones

Complex regular expressions can get hard to read quickly especially when there are many nested alternatives, and often, the same complex sub-expression can be part of a bigger expression several times.

The StringRegexAnnotator therefore provides a macro mechanism which allows for complex regular expression to get composed by simpler ones in steps: one can assign the simpler regular expressions to a macro variable and then use such variables in the final complex regular expression.

Here is an example where either ISO or "traditional" dates should get matched and where the year, month and day parts of the regular expression are more specific than in the examples above. Instead of copy-pasting those sub-expressions for the year, month and day into each rule, macro assignments are used:

rules7 = """
year=(19[0-9]{2}|20[0-9]{2})
month=(0[0-9]|10|11|12)
day=([012][0-9]|3[01])

// The ISO date:
|{{year}}-{{month}}-{{day}}
0 => Date  type="iso", year=G1, month=G2, day=G3

# The traditional way of writing a date:
|{{day}}/({{month}})/{{year}}
0 => Date  type="traditional", year=G3, month=G2, day=G1
"""

annt7 = StringRegexAnnotator(source=rules7, source_fmt="string")

doc7 = Document("""
A document that contains a date here: 2013-01-12 and also here: 14/02/1991. This should not 
get matched: 1833-12-21 and nor should this 45/03/2012 but this should 13/12/2012 and also
this, despite not being a valid data: 31/02/2000
""")

annt7(doc7)
doc7

.LAHFPNKGVO-row { width: 100%; display: flex; flex-direction: row; flex-wrap: nowrap; }

.LAHFPNKGVO-col { border: 1px solid grey; display: inline-block; min-width: 200px; padding: 5px; /* white-space: normal; / / white-space: pre-wrap; */ overflow-y: auto; }

.LAHFPNKGVO-hdr { font-size: 1.2rem; font-weight: bold; }

.LAHFPNKGVO-label { margin-bottom: -15px; display: block; }

.LAHFPNKGVO-input { vertical-align: middle; position: relative; *overflow: hidden; }

#LAHFPNKGVO-popup { display: none; color: black; position: absolute; margin-top: 10%; margin-left: 10%; background: #aaaaaa; width: 60%; height: 60%; z-index: 50; padding: 25px 25px 25px; border: 1px solid black; overflow: auto; }

.LAHFPNKGVO-selection { margin-bottom: 5px; }

.LAHFPNKGVO-featuretable { margin-top: 10px; }

.LAHFPNKGVO-fname { text-align: left !important; font-weight: bold; margin-right: 10px; } .LAHFPNKGVO-fvalue { text-align: left !important; } </style>

<script type="text/javascript">
let LAHFPNKGVO_data = {"annotation_sets": {"": {"name": "detached-from:", "annotations": [{"type": "Date", "start": 39, "end": 49, "id": 0, "features": {"type": "iso", "year": "2013", "month": "01", "day": "12"}}, {"type": "Date", "start": 65, "end": 75, "id": 1, "features": {"type": "traditional", "year": "02", "month": "02", "day": "14"}}, {"type": "Date", "start": 165, "end": 175, "id": 2, "features": {"type": "traditional", "year": "12", "month": "12", "day": "13"}}, {"type": "Date", "start": 223, "end": 233, "id": 3, "features": {"type": "traditional", "year": "02", "month": "02", "day": "31"}}], "next_annid": 4}}, "text": "\nA document that contains a date here: 2013-01-12 and also here: 14/02/1991. This should not \nget matched: 1833-12-21 and nor should this 45/03/2012 but this should 13/12/2012 and also\nthis, despite not being a valid data: 31/02/2000\n", "features": {}, "offset_type": "j", "name": ""} ; 
let LAHFPNKGVO_parms = {"presel_set": [], "presel_list": [], "cols4types": {}} ;
new gatenlpDocView(new gatenlpDocRep(LAHFPNKGVO_data, LAHFPNKGVO_parms), "LAHFPNKGVO-").init();
</script>

Combine with a String Gazetteer

In addition to the type of rules described above, there is a special rule which can be used to combine the regular expressions with StringGazetteer matching. The initialized StringGazetteer instances can be specified when creating the StringRegexAnnotator.

The rule consists of a single line of the form GAZETTEER => or GAZETTEER => feat1 = val1, feat2=val2 to assign some constant features (in addition to the features from the gazetteer entry and gazetteer list).

This examples illustrates this by additing a small string gazetteer to the previous example which matches the strings "date", "a date", "and", "also":

gazlist1 = [
    ("date", ),
    ("a date",),
    ("and",),
    ("also",),
]

gaz1 = StringGazetteer(source=gazlist1, source_fmt="gazlist")

rules8 = """
year=(19[0-9]{2}|20[0-9]{2})
month=(0[0-9]|10|11|12)
day=([012][0-9]|3[01])

// The ISO date:
|{{year}}-{{month}}-{{day}}
0 => Date  type="iso", year=G1, month=G2, day=G3

# The traditional way of writing a date:
|{{day}}/({{month}})/{{year}}
0 => Date  type="traditional", year=G3, month=G2, day=G1

# The rule to match the GAZETTEER
GAZETTEER => somefeature="some value"
"""

annt8 = StringRegexAnnotator(source=rules8, source_fmt="string", string_gazetteer=gaz1)

doc8 = Document("""
A document that contains a date here: 2013-01-12 and also here: 14/02/1991. This should not 
get matched: 1833-12-21 and nor should this 45/03/2012 but this should 13/12/2012 and also
this, despite not being a valid data: 31/02/2000
""")

annt8(doc8)
doc8

.JHUJIMGXLS-row { width: 100%; display: flex; flex-direction: row; flex-wrap: nowrap; }

.JHUJIMGXLS-col { border: 1px solid grey; display: inline-block; min-width: 200px; padding: 5px; /* white-space: normal; / / white-space: pre-wrap; */ overflow-y: auto; }

.JHUJIMGXLS-hdr { font-size: 1.2rem; font-weight: bold; }

.JHUJIMGXLS-label { margin-bottom: -15px; display: block; }

.JHUJIMGXLS-input { vertical-align: middle; position: relative; *overflow: hidden; }

#JHUJIMGXLS-popup { display: none; color: black; position: absolute; margin-top: 10%; margin-left: 10%; background: #aaaaaa; width: 60%; height: 60%; z-index: 50; padding: 25px 25px 25px; border: 1px solid black; overflow: auto; }

.JHUJIMGXLS-selection { margin-bottom: 5px; }

.JHUJIMGXLS-featuretable { margin-top: 10px; }

.JHUJIMGXLS-fname { text-align: left !important; font-weight: bold; margin-right: 10px; } .JHUJIMGXLS-fvalue { text-align: left !important; } </style>

<script type="text/javascript">
let JHUJIMGXLS_data = {"annotation_sets": {"": {"name": "detached-from:", "annotations": [{"type": "Lookup", "start": 26, "end": 32, "id": 0, "features": {"somefeature": "some value"}}, {"type": "Lookup", "start": 28, "end": 32, "id": 1, "features": {"somefeature": "some value"}}, {"type": "Date", "start": 39, "end": 49, "id": 2, "features": {"type": "iso", "year": "2013", "month": "01", "day": "12"}}, {"type": "Lookup", "start": 50, "end": 53, "id": 3, "features": {"somefeature": "some value"}}, {"type": "Lookup", "start": 54, "end": 58, "id": 4, "features": {"somefeature": "some value"}}, {"type": "Date", "start": 65, "end": 75, "id": 5, "features": {"type": "traditional", "year": "02", "month": "02", "day": "14"}}, {"type": "Lookup", "start": 118, "end": 121, "id": 6, "features": {"somefeature": "some value"}}, {"type": "Date", "start": 165, "end": 175, "id": 7, "features": {"type": "traditional", "year": "12", "month": "12", "day": "13"}}, {"type": "Lookup", "start": 176, "end": 179, "id": 8, "features": {"somefeature": "some value"}}, {"type": "Lookup", "start": 180, "end": 184, "id": 9, "features": {"somefeature": "some value"}}, {"type": "Date", "start": 223, "end": 233, "id": 10, "features": {"type": "traditional", "year": "02", "month": "02", "day": "31"}}], "next_annid": 11}}, "text": "\nA document that contains a date here: 2013-01-12 and also here: 14/02/1991. This should not \nget matched: 1833-12-21 and nor should this 45/03/2012 but this should 13/12/2012 and also\nthis, despite not being a valid data: 31/02/2000\n", "features": {}, "offset_type": "j", "name": ""} ; 
let JHUJIMGXLS_parms = {"presel_set": [], "presel_list": [], "cols4types": {}} ;
new gatenlpDocView(new gatenlpDocRep(JHUJIMGXLS_data, JHUJIMGXLS_parms), "JHUJIMGXLS-").init();
</script>

Using the StringRegexAnnotator API directly

The main methods of StringRegexAnnotator are:

append(source, source_fmt="file", list_features=None): to add one or more rule files/strings/rulelists to the annotator
find_all(..) to search some string using the stored rules and return a generator of Match objects

The find_all method can be useful when some string outside of a document should get processed, or when the matches need to get processed by code before they should get added as annotations to the document.

The following shows the result of calling find_all on the document text with the annotator configured above:

for m in annt8.find_all(doc8.text):
    print(m)

GazetteerMatch(start=26, end=32, match='a date', features={'somefeature': 'some value'}, type='Lookup')
GazetteerMatch(start=28, end=32, match='date', features={'somefeature': 'some value'}, type='Lookup')
GazetteerMatch(start=39, end=49, match='2013-01-12', features={'type': 'iso', 'year': '2013', 'month': '01', 'day': '12'}, type='Date')
GazetteerMatch(start=50, end=53, match='and', features={'somefeature': 'some value'}, type='Lookup')
GazetteerMatch(start=54, end=58, match='also', features={'somefeature': 'some value'}, type='Lookup')
GazetteerMatch(start=65, end=75, match='14/02/1991', features={'type': 'traditional', 'year': '02', 'month': '02', 'day': '14'}, type='Date')
GazetteerMatch(start=118, end=121, match='and', features={'somefeature': 'some value'}, type='Lookup')
GazetteerMatch(start=165, end=175, match='13/12/2012', features={'type': 'traditional', 'year': '12', 'month': '12', 'day': '13'}, type='Date')
GazetteerMatch(start=176, end=179, match='and', features={'somefeature': 'some value'}, type='Lookup')
GazetteerMatch(start=180, end=184, match='also', features={'somefeature': 'some value'}, type='Lookup')
GazetteerMatch(start=223, end=233, match='31/02/2000', features={'type': 'traditional', 'year': '02', 'month': '02', 'day': '31'}, type='Date')

Example use: GATE default tokenizer

The StringRegexAnnotator is used to implement the default_tokenizer, a tokenizer annotator which should work in the same way as the Java GATE DefaultTokenizer PR. The rules from the Java tokenizer have been directly converted into StringRegexAnnotator rules:

from gatenlp.lang.en.gatetokenizers import default_tokenizer, default_tokenizer_rules

print(default_tokenizer_rules)

#words#
// a word can be any combination of letters, including hyphens,
// but excluding symbols and punctuation, e.g. apostrophes
// Note that there is an alternative version of the tokeniser that
// treats hyphens as separate tokens


|(?:\p{Lu}(?:\p{Mn})*)(?:(?:\p{Ll}(?:\p{Mn})*)(?:(?:\p{Ll}(?:\p{Mn})*)|\p{Pd}|\p{Cf})*)*
0 =>  Token orth="upperInitial", kind="word", 

|(?:\p{Lu}(?:\p{Mn})*)(?:\p{Pd}|\p{Cf})*(?:(?:\p{Lu}(?:\p{Mn})*)|\p{Pd}|\p{Cf})+
0 =>  Token orth="allCaps", kind="word", 

|(?:\p{Ll}(?:\p{Mn})*)(?:(?:\p{Ll}(?:\p{Mn})*)|\p{Pd}|\p{Cf})*
0 =>  Token orth="lowercase", kind="word", 

// MixedCaps is any mixture of caps and small letters that doesn't
// fit in the preceding categories

|(?:(?:\p{Ll}(?:\p{Mn})*)(?:\p{Ll}(?:\p{Mn})*)+(?:\p{Lu}(?:\p{Mn})*)+(?:(?:\p{Lu}(?:\p{Mn})*)|(?:\p{Ll}(?:\p{Mn})*))*)|(?:(?:\p{Ll}(?:\p{Mn})*)(?:\p{Ll}(?:\p{Mn})*)*(?:\p{Lu}(?:\p{Mn})*)+(?:(?:\p{Lu}(?:\p{Mn})*)|(?:\p{Ll}(?:\p{Mn})*)|\p{Pd}|\p{Cf})*)|(?:(?:\p{Lu}(?:\p{Mn})*)(?:\p{Pd})*(?:\p{Lu}(?:\p{Mn})*)(?:(?:\p{Lu}(?:\p{Mn})*)|(?:\p{Ll}(?:\p{Mn})*)|\p{Pd}|\p{Cf})*(?:(?:\p{Ll}(?:\p{Mn})*))+(?:(?:\p{Lu}(?:\p{Mn})*)|(?:\p{Ll}(?:\p{Mn})*)|\p{Pd}|\p{Cf})*)|(?:(?:\p{Lu}(?:\p{Mn})*)(?:\p{Ll}(?:\p{Mn})*)+(?:(?:\p{Lu}(?:\p{Mn})*)+(?:\p{Ll}(?:\p{Mn})*)+)+)|(?:(?:(?:\p{Lu}(?:\p{Mn})*))+(?:(?:\p{Ll}(?:\p{Mn})*))+(?:(?:\p{Lu}(?:\p{Mn})*))+)
0 =>  Token orth="mixedCaps", kind="word", 

|(?:\p{Lo}|\p{Mc}|\p{Mn})+
0 => Token kind="word", type="other", 

#numbers#
// a number is any combination of digits
|\p{Nd}+
0 => Token kind="number", 

|\p{No}+
0 => Token kind="number", 

#whitespace#
|(?:\p{Zs}) 
0 => SpaceToken kind="space", 

|(?:\p{Cc}) 
0 => SpaceToken kind="control", 

#symbols#
|(?:\p{Sk}|\p{Sm}|\p{So}) 
0 =>  Token kind="symbol", 

|\p{Sc} 
0 =>  Token kind="symbol", symbolkind="currency", 

#punctuation#
|(?:\p{Pd}|\p{Cf}) 
0 => Token kind="punctuation", subkind="dashpunct", 

|(?:\p{Pc}|\p{Po})
0 => Token kind="punctuation", 

|(?:\p{Ps}|\p{Pi}) 
0 => Token kind="punctuation", position="startpunct", 

|(?:\p{Pe}|\p{Pf}) 
0 => Token kind="punctuation", position="endpunct",

doc = Document("""
This is a short document. Has miXedCaps and ALLUPPER and 1234 and hyphen-word. 
Also something after a new line. And another sentence. A float 3.4123 and a code XZ-2323-a.
""")

default_tokenizer(doc)
doc

.MOXKMUOVRF-row { width: 100%; display: flex; flex-direction: row; flex-wrap: nowrap; }

.MOXKMUOVRF-col { border: 1px solid grey; display: inline-block; min-width: 200px; padding: 5px; /* white-space: normal; / / white-space: pre-wrap; */ overflow-y: auto; }

.MOXKMUOVRF-hdr { font-size: 1.2rem; font-weight: bold; }

.MOXKMUOVRF-label { margin-bottom: -15px; display: block; }

.MOXKMUOVRF-input { vertical-align: middle; position: relative; *overflow: hidden; }

#MOXKMUOVRF-popup { display: none; color: black; position: absolute; margin-top: 10%; margin-left: 10%; background: #aaaaaa; width: 60%; height: 60%; z-index: 50; padding: 25px 25px 25px; border: 1px solid black; overflow: auto; }

.MOXKMUOVRF-selection { margin-bottom: 5px; }

.MOXKMUOVRF-featuretable { margin-top: 10px; }

.MOXKMUOVRF-fname { text-align: left !important; font-weight: bold; margin-right: 10px; } .MOXKMUOVRF-fvalue { text-align: left !important; } </style>

<script type="text/javascript">
let MOXKMUOVRF_data = {"annotation_sets": {"": {"name": "detached-from:", "annotations": [{"type": "SpaceToken", "start": 0, "end": 1, "id": 0, "features": {"kind": "control"}}, {"type": "Token", "start": 1, "end": 5, "id": 1, "features": {"orth": "upperInitial", "kind": "word"}}, {"type": "SpaceToken", "start": 5, "end": 6, "id": 2, "features": {"kind": "space"}}, {"type": "Token", "start": 6, "end": 8, "id": 3, "features": {"orth": "lowercase", "kind": "word"}}, {"type": "SpaceToken", "start": 8, "end": 9, "id": 4, "features": {"kind": "space"}}, {"type": "Token", "start": 9, "end": 10, "id": 5, "features": {"orth": "lowercase", "kind": "word"}}, {"type": "SpaceToken", "start": 10, "end": 11, "id": 6, "features": {"kind": "space"}}, {"type": "Token", "start": 11, "end": 16, "id": 7, "features": {"orth": "lowercase", "kind": "word"}}, {"type": "SpaceToken", "start": 16, "end": 17, "id": 8, "features": {"kind": "space"}}, {"type": "Token", "start": 17, "end": 25, "id": 9, "features": {"orth": "lowercase", "kind": "word"}}, {"type": "Token", "start": 25, "end": 26, "id": 10, "features": {"kind": "punctuation"}}, {"type": "SpaceToken", "start": 26, "end": 27, "id": 11, "features": {"kind": "space"}}, {"type": "Token", "start": 27, "end": 30, "id": 12, "features": {"orth": "upperInitial", "kind": "word"}}, {"type": "SpaceToken", "start": 30, "end": 31, "id": 13, "features": {"kind": "space"}}, {"type": "Token", "start": 31, "end": 40, "id": 14, "features": {"orth": "mixedCaps", "kind": "word"}}, {"type": "SpaceToken", "start": 40, "end": 41, "id": 15, "features": {"kind": "space"}}, {"type": "Token", "start": 41, "end": 44, "id": 16, "features": {"orth": "lowercase", "kind": "word"}}, {"type": "SpaceToken", "start": 44, "end": 45, "id": 17, "features": {"kind": "space"}}, {"type": "Token", "start": 45, "end": 53, "id": 18, "features": {"orth": "allCaps", "kind": "word"}}, {"type": "SpaceToken", "start": 53, "end": 54, "id": 19, "features": {"kind": "space"}}, {"type": "Token", "start": 54, "end": 57, "id": 20, "features": {"orth": "lowercase", "kind": "word"}}, {"type": "SpaceToken", "start": 57, "end": 58, "id": 21, "features": {"kind": "space"}}, {"type": "Token", "start": 58, "end": 62, "id": 22, "features": {"kind": "number"}}, {"type": "SpaceToken", "start": 62, "end": 63, "id": 23, "features": {"kind": "space"}}, {"type": "Token", "start": 63, "end": 66, "id": 24, "features": {"orth": "lowercase", "kind": "word"}}, {"type": "SpaceToken", "start": 66, "end": 67, "id": 25, "features": {"kind": "space"}}, {"type": "Token", "start": 67, "end": 78, "id": 26, "features": {"orth": "lowercase", "kind": "word"}}, {"type": "Token", "start": 78, "end": 79, "id": 27, "features": {"kind": "punctuation"}}, {"type": "SpaceToken", "start": 79, "end": 80, "id": 28, "features": {"kind": "space"}}, {"type": "SpaceToken", "start": 80, "end": 81, "id": 29, "features": {"kind": "control"}}, {"type": "Token", "start": 81, "end": 85, "id": 30, "features": {"orth": "upperInitial", "kind": "word"}}, {"type": "SpaceToken", "start": 85, "end": 86, "id": 31, "features": {"kind": "space"}}, {"type": "Token", "start": 86, "end": 95, "id": 32, "features": {"orth": "lowercase", "kind": "word"}}, {"type": "SpaceToken", "start": 95, "end": 96, "id": 33, "features": {"kind": "space"}}, {"type": "Token", "start": 96, "end": 101, "id": 34, "features": {"orth": "lowercase", "kind": "word"}}, {"type": "SpaceToken", "start": 101, "end": 102, "id": 35, "features": {"kind": "space"}}, {"type": "Token", "start": 102, "end": 103, "id": 36, "features": {"orth": "lowercase", "kind": "word"}}, {"type": "SpaceToken", "start": 103, "end": 104, "id": 37, "features": {"kind": "space"}}, {"type": "Token", "start": 104, "end": 107, "id": 38, "features": {"orth": "lowercase", "kind": "word"}}, {"type": "SpaceToken", "start": 107, "end": 108, "id": 39, "features": {"kind": "space"}}, {"type": "Token", "start": 108, "end": 112, "id": 40, "features": {"orth": "lowercase", "kind": "word"}}, {"type": "Token", "start": 112, "end": 113, "id": 41, "features": {"kind": "punctuation"}}, {"type": "SpaceToken", "start": 113, "end": 114, "id": 42, "features": {"kind": "space"}}, {"type": "Token", "start": 114, "end": 117, "id": 43, "features": {"orth": "upperInitial", "kind": "word"}}, {"type": "SpaceToken", "start": 117, "end": 118, "id": 44, "features": {"kind": "space"}}, {"type": "Token", "start": 118, "end": 125, "id": 45, "features": {"orth": "lowercase", "kind": "word"}}, {"type": "SpaceToken", "start": 125, "end": 126, "id": 46, "features": {"kind": "space"}}, {"type": "Token", "start": 126, "end": 134, "id": 47, "features": {"orth": "lowercase", "kind": "word"}}, {"type": "Token", "start": 134, "end": 135, "id": 48, "features": {"kind": "punctuation"}}, {"type": "SpaceToken", "start": 135, "end": 136, "id": 49, "features": {"kind": "space"}}, {"type": "Token", "start": 136, "end": 137, "id": 50, "features": {"orth": "upperInitial", "kind": "word"}}, {"type": "SpaceToken", "start": 137, "end": 138, "id": 51, "features": {"kind": "space"}}, {"type": "Token", "start": 138, "end": 143, "id": 52, "features": {"orth": "lowercase", "kind": "word"}}, {"type": "SpaceToken", "start": 143, "end": 144, "id": 53, "features": {"kind": "space"}}, {"type": "Token", "start": 144, "end": 145, "id": 54, "features": {"kind": "number"}}, {"type": "Token", "start": 145, "end": 146, "id": 55, "features": {"kind": "punctuation"}}, {"type": "Token", "start": 146, "end": 150, "id": 56, "features": {"kind": "number"}}, {"type": "SpaceToken", "start": 150, "end": 151, "id": 57, "features": {"kind": "space"}}, {"type": "Token", "start": 151, "end": 154, "id": 58, "features": {"orth": "lowercase", "kind": "word"}}, {"type": "SpaceToken", "start": 154, "end": 155, "id": 59, "features": {"kind": "space"}}, {"type": "Token", "start": 155, "end": 156, "id": 60, "features": {"orth": "lowercase", "kind": "word"}}, {"type": "SpaceToken", "start": 156, "end": 157, "id": 61, "features": {"kind": "space"}}, {"type": "Token", "start": 157, "end": 161, "id": 62, "features": {"orth": "lowercase", "kind": "word"}}, {"type": "SpaceToken", "start": 161, "end": 162, "id": 63, "features": {"kind": "space"}}, {"type": "Token", "start": 162, "end": 165, "id": 64, "features": {"orth": "allCaps", "kind": "word"}}, {"type": "Token", "start": 165, "end": 169, "id": 65, "features": {"kind": "number"}}, {"type": "Token", "start": 169, "end": 170, "id": 66, "features": {"kind": "punctuation", "subkind": "dashpunct"}}, {"type": "Token", "start": 170, "end": 171, "id": 67, "features": {"orth": "lowercase", "kind": "word"}}, {"type": "Token", "start": 171, "end": 172, "id": 68, "features": {"kind": "punctuation"}}, {"type": "SpaceToken", "start": 172, "end": 173, "id": 69, "features": {"kind": "control"}}], "next_annid": 70}}, "text": "\nThis is a short document. Has miXedCaps and ALLUPPER and 1234 and hyphen-word. \nAlso something after a new line. And another sentence. A float 3.4123 and a code XZ-2323-a.\n", "features": {}, "offset_type": "j", "name": ""} ; 
let MOXKMUOVRF_parms = {"presel_set": [], "presel_list": [], "cols4types": {}} ;
new gatenlpDocView(new gatenlpDocRep(MOXKMUOVRF_data, MOXKMUOVRF_parms), "MOXKMUOVRF-").init();
</script>

Notebook last updated

import gatenlp
print("NB last updated with gatenlp version", gatenlp.__version__)

NB last updated with gatenlp version 1.0.8a1

" + fname + "

" + fval + "

python-gatenlp/docs/stringregex.md at main · GateNLP/python-gatenlp · GitHub

Uh oh!

String Regex Annotator Tutorial

Creating the Annotator

Create from a string with rules

The rules file/string format

Using groups

Adding features to annotations

Having more than one rule

Longest match only

Choosing matching rules

Overlapping matches

Macros: creating complex regular expressions from simpler ones

Combine with a String Gazetteer

Using the StringRegexAnnotator API directly

Example use: GATE default tokenizer

Notebook last updated

python-gatenlp/docs/stringregex.md at main · GateNLP/python-gatenlp · GitHub

Uh oh!

Expand file tree

Breadcrumbs

stringregex.md

Latest commit

History

Breadcrumbs

stringregex.md

File metadata and controls

String Regex Annotator Tutorial

Creating the Annotator

Create from a string with rules

The rules file/string format

Using groups

Adding features to annotations

Having more than one rule

Longest match only

Choosing matching rules

Overlapping matches

Macros: creating complex regular expressions from simpler ones

Combine with a String Gazetteer

Using the StringRegexAnnotator API directly

Example use: GATE default tokenizer

Notebook last updated