noodbabstract.coffee

Jump To … +

server.coffee src/actionknob.coffee src/autosem.coffee src/bitbucket_kba.coffee src/browserlog.coffee src/datareduction.coffee src/dci.coffee src/dciknob.coffee src/deeseeeye.coffee src/dnd.coffee src/doof.coffee src/formurla-mngr.coffee src/fractalpanel.coffee src/fractalpanel_test.coffee src/front.coffee src/ingestor.coffee src/kbabitbucket.coffee src/knobctrl.coffee src/lib_test.coffee src/nanoclock.coffee src/noodb.coffee src/noodbabstract.coffee src/noodbbrowser.coffee src/noodbbrowser_test.coffee src/noodbsec.coffee src/noorauth.coffee src/noorplugin.coffee src/noorquery.coffee src/noorvm.coffee src/noorwrite.coffee src/quadparser.coffee src/quadparsern3.coffee src/rbac.coffee src/reactor.coffee src/rebase.coffee src/rsrcidx.coffee src/sandboxactions.coffee src/screen_ctx.coffee src/spogi.coffee src/tabular_widget.coffee src/visctrl.coffee src/voicesknob.coffee src/whowhen.coffee src/xsd2native.coffee

noodbabstract.coffee

_ = (window? and window._) or require('underscore')
RsrcDb = require('./rsrcidx').RsrcDb
PrefixDb = require('./rsrcidx').PrefixDb
Spogi = require("./spogi").Spogi
WhoWhen = require("./whowhen").WhoWhen
RdfObject = require("./quadparser").RdfObject
rebase = require("./rebase")
int_to_base = rebase.int_to_base

class Query
  constructor: (@terms, @select_spec) ->

  isSatisfiedBy: (spogi) ->
    satisfied = true
    for term in @terms
      list = []
      for spog_or_i, one_or_list of term
        if _.isArray(one_or_list)
          list = one_or_list
        else
          list = [one_or_list]
        rsrc = spogi[spog_or_i]
        val = rsrc.key()
        if list.lastIndexOf(val) is -1
          satisfied = false
          break
      if not list.length

TODO figure out what to do when the query term is empty

        console.log "isSatisfiedBy() query term is empty", term
    return satisfied

class NooDBAbstract
  warn_once: (warning) ->
    if not @_warnings
      @_warnings = {}
    if @_warnings[warning]
      @_warnings[warning]++
    else
      @_warnings[warning] = 1
      @log.warning(warning)

console.warn(warning)

  report: (report_name, why) ->
    @log.debug("report(#{why})")

why.TYP=1 @allege(‘nrn:’+report_name, ‘dc:comment’, why, ‘nrn:test_db’)

  set_log_level_by_num: (levelNo) ->
    @log.level = levelNo

  name_it: (thing, name) ->
    if thing?
      if thing._name?
        throw "thing._name = '#{thing._name}' so can not assign '#{name}'"
      thing._name = name

¶

else throw “there is NO thing to name ‘#{name}’”
```
  resolve_defaults: (defaults, args, Log) ->
```

Precedence is: args > defaults > this

    _.extend(this, _.extend(defaults, args))
    if not @log
      @log = new Log(@log_level) # https://github.com/tj/log.js
    @log.info("log_level: #{@log_level}")

  set_server_start_time: (date) ->
    @server_start_time = date

  init_db: (rd_ctx, init_callback) ->

@name_it(init_callback, “init_callback”)

    @set_server_start_time(@clock.asDate())
    @read_count = 0
    @read_warn_last_time = @clock.longAgo()

    open_callback = () =>
      if init_callback?
        init_callback()
      return
    @name_it(open_callback, "open_callback")

    read_callback = () =>
      @server_log("nrn:serverLoadedSpogiCount", [@read_count])
      @server_log("nrn:serverStartedAt", [@clock.asDate().toISOString()])
      @open_db(open_callback)
      return
    @name_it(read_callback, "read_callback")
    @read_db(@fname, rd_ctx, read_callback)

  get_array_of_random: ->
    throw new Error("get_array_of_random should be implemented in subclasses")

  next_entity_id: (digits) ->
    digits ?= 10
    return rebase.symbol_from_bytes(@get_array_of_random(digits))

  read_db: (fname, read_more, fname_queue, callback) ->
    @synthetic_key_factory_init(@clock) # TODO better is file change date

  synthetic_key_factory_init: (@synthetic_key_factory_clock) ->

  synthetic_key_factory_last: (last_key) ->
    parts = last_key.split("_")
    last_sec_baseXX = parts[2]
    last_nsec_baseXX = parts[3]
    @clock.last.sec = base_to_int(last_sec_baseXX)
    @clock.last.nsec = base_to_int(last_nsec_baseXX)

  synthetic_key_factory_next: () ->
    ss = @get_server_session()
    now = @clock.now()
    new WhoWhen().build(
        ss.user_symbol
        ss.session_no
        now.sec
        now.nsec).repr()

  now_baseXX: () ->
    return int_to_base(@clock.now().sec)

  build_indices: ->
    @_query_listeners = {} # TODO ren

    @prefixdb =  new PrefixDb()
    @by_subj =   new RsrcDb(@prefixdb)
    @by_pred =   new RsrcDb(@prefixdb)
    @by_obj =    new RsrcDb(@prefixdb)
    @by_graph =  new RsrcDb(@prefixdb)
    @by_id =     new RsrcDb(@prefixdb)
    @by =
      s: @by_subj
      p: @by_pred
      o: @by_obj
      g: @by_graph
      i: @by_id

  preload_prefixes: () ->
    for k, v of @universal_prefixes
      @declare_prefix(k, v)
    if @preloadable_prefixes
      for k, v of @preloadable_prefixes
        @declare_prefix(k, v)

  make_spogi_from_penta: (penta) ->
    console.log "make_spogi_from_penta() is DEPRECATED"

A penta is JS object with keys ‘s,p,o,g,i’ and values either RdfUri or RdfObject as appropriate though .i is a string console.log “====> make_spogi_from_penta() o_isUri:#{o_isUri} o_value:#{o_value}”

    spogi = new Spogi(
        @by_subj.getOrCreate(penta.s.raw, true)
        @by_pred.getOrCreate(penta.p.raw, true)
        @by_obj.getOrCreate(penta.o.value, penta.o.isUri())
        @by_graph.getOrCreate(penta.g.raw, true)
        @by_id.getOrCreate(penta.i, false))
    return spogi

  make_spogi_from_quint: (quint, context) ->

¶

A quint is a 5-tuple with all values being strings in the order s,p,o,g,i and with values which are URIs for s,p,g (and sometimes o) and a NID for i. Sometimes o? When the third term, o, satisfies o.match(/^[“-+\d]/) ie when o is a literal value because it is like: “whatever”@en “cafebabe”^^xsd:anyThing “whatever” 3.14159 +3.14159 -3 3 Incidentally, NIDs (eg: A2Xef9_f934z_94aS_23r) never start like a literal o either.

Note also that a url can start with : as a prefix character
¶

TODO 1) ensure that quints as defined above are passed to this method 2) ensure that RsrcDb..getOrCreate() conforms to this new interface TODO clarify: this is either context or write_ctx
```
    context ?= @make_default_context()
```

@log.debug(“make_spogi_from_quint() quint:”, quint) isObjUri = quint[2].match(

    spogi = new Spogi(
        @by_subj.getOrCreate(quint[0], null, context)
        @by_pred.getOrCreate(quint[1], null, context)
        @by_obj.getOrCreate(quint[2], null, context)
        @by_graph.getOrCreate(quint[3], null, context)
        @by_id.getOrCreate(quint[4], null, context))

¶

@log.debug “ spogi:”, typeof spogi #spogi?[‘constructor’]?[‘name’] @log.debug(“ <– spogi:”, spogi and spogi.asLine() or “d’oh”)
```
    return spogi

  index: (quint, context) ->
    try_writing = (context? and context.try_writing) or false
```

the id should be unique so deal with it first in case of collision

    if /ontology YAGO3/.exec(quint[2])
      console.log("THE FIRST CHAR OF '...ontology YAGO3...' IS <#{quint[2][0]}>")
    key = quint[4]
    if @by_id.exists(key)
      existing = @by_id.getOnly(key)

¶

throw new Error “should handle id collision: #{key}”
```
      if existing.eqlQuint(quint)
```

TODO replace with semantic logging

        msg = "index(#{key}) ignored exact duplicate"
        @log.info(msg)
        return
      else
        e_str = existing.asTTL()
        q_str = ""+quint
        msg = "index(#{key}) failed: key match but not content. #{e_str} <> #{q_str}"
        error_on_nonmatching_id_collision = false
        if error_on_nonmatching_id_collision
          throw new Error(msg)
        else
          @log.warning(msg)
          return

    context = context or quint[5] # REVIEW(smurp) should opts be dominating (or replacing) quint[5]
    spogi = @make_spogi_from_quint(quint, context)
    if try_writing and @writeable_path?
      @persist(spogi)

    processQueries = true
    spogi.s.addObj(spogi, processQueries)
    spogi.p.addObj(spogi, processQueries)
    spogi.o.addObj(spogi, processQueries)
    spogi.g.addObj(spogi, processQueries)
    spogi.i.addObj(spogi, processQueries)
    return spogi

  declare_prefix: (prefix, expansion) ->
    try
      @prefixdb.add_prefix(prefix, expansion)
    catch e
      if not e.message.match('already been defined') # ignore these 'no harm' collisions
        throw e

else console.warn e.message

  allege_line: (line, sess, date) ->
    quint = parseQuadLineToQuint(line)
    quint[4] = undefined # differentiate between parseQuadLine and parseQuintLine
    @allege(quint[0], quint[1], quint[2], quint[3], sess, date)

  allege: (s, p, o, g, sess, date, context) ->
    if typeof o isnt 'string' # TODO find a more correct way to see if o is a String
      o = "#{o}"
    if undefined in [s, p, o]
      throw new Error "everything must be defined of <s:#{s}> <p:#{p}> <o:#{o}>"
    if not sess?
      sess = @get_server_session()
    if date?
      now = date_to_now(date)
    else
      now = @clock.now()
    context ?= @make_default_write_ctx()
    ww = new WhoWhen().build(
                        sess.user_symbol
                        sess.session_no
                        now.sec
                        now.nsec)
    g = g ? @default_graph
    @index([s, p, o, g, ww.toSafeCURIE()], context) # REVIEW(smurp) context smells

  select: (lol, select_spec) ->

select() takes lol - a list of lists and select_spec which: all, first, last columns (optional): A list of characters indicating which columns to return (s,p,o,g,i) If columns is provided then the return values is a list of lists and if it is not provided then a list of spogi is returned. flat (optional): A boolean to indicate that the return value should be a single list of the one column indicated in the columns setting if @log.level is 7 @log.debug “crushing it” for list,j in lol @log.debug(“line #{j}”, (item.key() for item in list) )

    @log.debug("select() lol.length: #{lol.length} select_spec:",select_spec)
    @log.debug("  lol:",[l for l in lol], lol)
    if global.TRACE
      console.log("    lol:")
      console.log(lol)
      console.log("    select_spec:")
      console.log(select_spec)
    if select_spec.columns? and select_spec.columns.length > 0
      @log.debug("projector returns listOfLists")
      projector = (listOfSpogi) ->
        flatten = select_spec.columns.length is 1 and select_spec.flat
        listOfLists = []
        for spogi in listOfSpogi
          row = []
          for columnId in select_spec.columns
            row.push spogi[columnId].key()
          if flatten
            listOfLists.push(row[0])
          else
            listOfLists.push(row)
        return listOfLists
    else
      @log.debug("projector is NOOP", lol.length, lol[lol.length-1])
      projector = (listOfSpogi) ->
        return listOfSpogi
    switch select_spec.which
      when 'all'
        if not @instrumented
          return projector(_.intersection.apply(this,lol))
        else
          @log.debug(lol)
          @log.debug("lol: \n  " + lol.join(";\n  "))
          pass1 = projector(_.intersection.apply(this,lol))
          @log.debug("intersection: \n  " + pass1.join(";\n  "))
          return pass1
      when 'first'
        return projector(_.first(_.intersection.apply(this, lol)))
      when 'last'
        return projector(_.last(_.intersection.apply(this, lol)))
      when null
        throw "NooDB.select(list_of_lists, {which: 'all'|'first'|'last'})"

  q: (s, p, o, g, i) ->
    @log.debug "q(#{s}, #{p}, #{o}, #{g}, #{i})"
    terms = []

FIXME this is so ugly! (but maybe it is the most efficient? probably) Wanted to do something like: terms = .object(.zip(“spogi”.split(‘’),arguments)) but mumble mumble

    if s?
      terms.push({s: s})
    if p?
      terms.push({p: p})
    if o?
      terms.push({o: o})
    if g?
      terms.push({g: g})
    if i?
      terms.push({i: i})

    server = @
    canned_query =
      terms: terms
      run: _.bind(server.query, server, terms)
      last: () ->
        @run({which: 'last'})
      first: () ->
        @run({which: 'first'})
      all: () ->
        @run()

  make_default_context: ->

return {blank_prefix: null} return {blank_prefix: ‘_’}

    return {blank_prefix: 'DEFAULT_BLANK_PREFIX'}

  make_default_write_ctx: ->
    @make_default_context()

  query: (terms, select_spec) ->

¶

supported queries: query([{s: “mailto:bob”}]) all spogi with mailto:bob as subj query([{s: [“mailto:bob”,”mailto:betty”}]) all spogi with either mailto:bob OR mailto:betty as subj query([{s: “nrn:fred”, o: “nrn:barney”}]) all spogi with subj=fred OR obj=barney query([{s: “nrn:plato”}, {p: “nrn:ate”}, {o: “nrn:hemlock”}]) all spogi where plato AND ate AND hemlock query([{s: “nrn:Pagliacci”}, {p: “nrn:laughed”}], {which: ‘last’}) the last (latest) spogi where Pagliacci AND laughed query([{s: “nrn:nuke”}, {p: “nrn:explodedAtTime”}], {which: ‘first’}) the first (earliest) spogi where a nuke AND exploded
```
    select_spec = select_spec ? {}
    select_spec.which ?= "all"
    select_spec.context ?= @make_default_context()
    lol = [] # list of lists
    @log.debug("query()")
    for term in terms
      l = []
      @log.debug("  term:", term)
      for k, one_or_list of term
        if _.isArray(one_or_list)
          list = one_or_list
        else
          list = [one_or_list]
        for v in list
```

safe_curie_v = @prefixdb.toSafeCURIE(v) canonical_v = @by[k]get_canonical_form_typ_pair(v)[0] console.log(“safe_curie_v:”, safe_curie_v) l = l.concat(@by[k].getAll(safe_curie_v))

          if global.TRACE
            console.log("query() processing term k='#{k}' v='#{v}'")
          more = @by[k].getAll(v, null, select_spec.context)
          @log.debug("  l.concat(@by.#{k}.getAll('#{v}')", more)
          l = l.concat(more)

@log.debug k, Object.keys(@by[k].mbrs)

          @log.debug("  l:", l)
      if not l.length

we can short circuit this failed query because this term is empty

        @log.debug("  SHORT CIRCUIT", l, lol)
        @select([[]], select_spec) # let select prepare the result
      lol.push(l)
    @log.debug("  -> lol:", [l.constructor.name for l in lol], lol)
    return @select(lol, select_spec)

  exists: (terms) ->
    resp = @query(terms, {which: 'first'})
    if not resp?
      return false
    else
      return true

¶

http://rdflib.readthedocs.org/en/latest/univrdfstore.html#formula-context-interfaces
```
  contexts: ->
```

TODO this should be async TODO should accept a triple to constrain response

    retval = []
    for graph of @by_graph.mbrs
      retval.push graph
    return retval

  on_do: (query_terms, callback) ->
    icl = new InsecureCallbackListener(callback)
    @subscribe_listener(icl, query_terms)

TODO decide whether it should be a Query or just a ‘terms’ presumably a select_spec is needed too, especially for queries where the response spogi count should be 1, eg .last(), first() etc

  at_log_level_run: (log_level, func) ->
    orig_log_level = @log.level
    @set_log_level_by_num(log_level)
    retval = func()
    @set_log_level_by_num(orig_log_level)
    return retval

  subscribe: (query_terms) ->
    @on_do query_terms, (spogi) =>
      @noodb.log("drop on floor:", spogi.toString())

a NOOP subscription, ie unconditionally subscribe to the query_terms

  subscribe_listener: (listener, qry) ->
    symbol = toSymbol(qry)
    @log.debug("symbol:",symbol)
    q4l = @_query_listeners[symbol]
    if not q4l?
      q4l = new QueryForListeners(qry, symbol, @)
      @_query_listeners[symbol] = q4l
    q4l.add_listener(listener)
    @log.debug('  q4l.listeners:', q4l.listeners, ".query:",q4l.query)
    permission_p = 'nrn:readableBy'
    @query(q4l.query.terms, {which: 'all'}).forEach (spogi) =>
      @log.debug("subscribe() initial query spogi:", spogi.asRaw())
      q4l.sendIfPermitted(spogi, permission_p, listener)
    return

  unsubscribe_listener: (listener, qry) -> # TODO ensure this gets called eg when a visualization closes
    symbol = toSymbol(qry)
    q4l = @_query_listeners[symbol]
    if q4l?
      q4l.remove_listener(listener)

  dump_matching: (regex) ->
    for k, v of @by_id.mbrs
      if not regex? or k.match(regex)
        console.log(k, v.occs[0].toString())

  extractDateFromCurie: (curie) ->
    return (new WhoWhen().parse(curie).getDate())

class QueryForListeners

TODO organize QueryForListeners into a ‘grove’ so

  constructor: (terms, @symbol, @noodb) ->
    @query = new Query(terms)
    @listeners = []
    @registerQueryOnFirstResource()

  registerQueryOnFirstResource: ->

Find the resources mentioned in the first term of the query and hang this q4l off it so when satisfying spogi are indexed they can have their listeners notified.

    term = @query.terms[0]
    if term?
      for spog_or_i, one_or_list of term
        list = []
        if Array.isArray(one_or_list)
          list = one_or_list
        else
          list = [one_or_list]
        for rsrc_key in list
          rsrc = @noodb.by[spog_or_i].getOrCreate(rsrc_key) # Rsrc might not exist yet, so OrCreate
          if rsrc?
            @noodb.log.info("ADDING QUERY FOR LISTENERS for rsrc_key:",rsrc_key)
            rsrc.addQueryForListeners(@)
          else
            @noodb.log.error("QueryForListener.registerQueryOnFirstResource() no Rsrc for key: #{rsrc_key}")
    else
      @noodb.log.error("Query has no terms")

  add_listener: (listener) ->
    @listeners.push(listener)

  remove_listener: (listener) ->
    idx = @listeners.indexOf(listener)
    if idx
      @listeners.splice(idx, 1)

  sendIfSatisfies: (spogi) ->

@noodb.log.debug “sendIfSatisfies() spogi: #{spogi.toString()}” console.log “sendIfSatisfies() spogi: #{spogi.toString()}”

    if @query.isSatisfiedBy(spogi)
      @listeners.forEach (listener) =>
        permission_p = 'nrn:readableBy'
        retval = @sendIfPermitted(spogi, permission_p, listener)

¶

TODO should retire listeners if their select_spec is satisfied (eg: ‘first’)
```
  sendIfPermitted: (spogi, permission_p, listener) ->
```

for k,v of listener console.log k console.log “listener”,listener listener.send spogi

    if listener.hasPermission(spogi, permission_p, @noodb)
      listener.send(spogi)
      return true
    return false

class AbstractListener
  getUser: (noodb) ->
    return

class InsecureListener extends AbstractListener
  hasPermission: (spogi, permission_p, noodb) ->
    true

class InsecureCallbackListener extends InsecureListener
  constructor: (@callback) ->
    super()
    @id = (new Date()).toISOString()
  get_id: ->
    return @id
  send: (spogi) ->
    @callback(spogi)

class SecureListener extends AbstractListener
  constructor: ->
    super()
    @perms = {} # this is a cache of memoized responses for hasPermission

userHasPermissionOnGraph: (noodb, g_key) ->

  userHasPermissionOnGraph: (noodb, permission_p, g_key) ->

Return false if there is no user or that user can not read the graph either because they have personal or group privileges to it. TODO add graphIsReadableByGroupUserIsIn()

    retval = false
    user = @getUser(noodb)
    noodb.log.info("user:",JSON.stringify(user))
    o_list = ['nrn:public','nrn:unlisted']
    if user?
      o_list.push(user.userCURIE)

noodb.log.alert “o_list:”, o_list

    public_unlisted_or_the_user = noodb.exists [
        s: g_key
      ,
        p: permission_p
      ,
        o: o_list
      ,
        g:'nrn:permissionsKB'
      ]
    if public_unlisted_or_the_user
      return true
    if not user?
      why =  "userHasPermissionOnGraph(#{permission_p}, #{g_key}) ==> false, BECAUSE there is no user"
      noodb.report('SecureListener', why)
      return false
    if not user.groups?

if we do not yet know the user groups, cache them now

      user.groups = noodb.query([
          s: user.userCURIE
        ,
          p: 'nrn:inGroup'
        ,
          g: 'nrn:permissionsKB'
        ], {which:'all', columns:['o'], flat:true})


    if user.groups.length > 0

The user is in some groups, so is g_key readableBy any of them?

      noodb.log.debug("#{user.userCURIE} is in #{user.groups}")
      retval = noodb.exists [
          s: g_key
        ,
          p: permission_p
        ,
          o: user.groups
        ,
          g:'nrn:permissionsKB'
        ]

¶

else

The user is in no groups, so can not be permitted because of membership.

why = “userHasPermissionOnGraph(#{permission_p}, #{g_key}) ==> false, BECAUSE user #{user.userCURIE} is in no groups” noodb.report(‘SecureListener’, why)
```
    if not retval
```

So far no positive permission has been found. Check to see if the container this KB was found in gives permission

      retval = @userHasPermissionOnGraphContainer(noodb, permission_p, g_key, user)

    if retval is false
      why = "userHasPermissionOnGraph(#{permission_p}, #{g_key}) ==> false, BECAUSE user #{user.userCURIE} groups not permitted"
      noodb.report('SecureListener', why)
    return retval

  userHasPermissionOnGraphContainer: (noodb, permission_p, g_key, user) ->
    p = 'nrn:inheritsPermissionsFrom'
    inheritedPermissions = noodb.query([
        s: g_key
      ,
        p: p
      ,
        g: 'nrn:permissionsKB'
      ])
    containers = {}
    for perm in inheritedPermissions
      container = perm.o.key()
      if not containers[container] # has it not been tested before?
        if @userHasPermissionOnGraph(noodb, permission_p, container)
          return true # permission found, bail out
        containers[container] = true # mark as tested, continue
    return false

  hasPermission: (spogi, permission_p, noodb) ->
    user = @getUser(noodb)
    g_key = spogi.g.key()
    if not @perms[permission_p]? # eg permission_p = 'nrn:readableBy'
      @perms[permission_p] = {}
    perm = @perms[permission_p][g_key]
    if user?
      userId = user.userId
    else
      userId = 'undefined'
    noodb.log.info("hasPermission(#{spogi.toString()}) userId:#{userId} perm:#{perm}")
    if not perm?
      perm = @userHasPermissionOnGraph(noodb, permission_p, g_key)
      @perms[permission_p][g_key] = perm
    return perm

keys = 'spogi'.split('')

toSymbol = (query) ->

¶

Purpose: Return concatented property/value pairs of query to use as JS identifier Notes: A query is just a bare object with possible keys: s,p,o,g,i The keys, if present should always be concated in the same order console.log(“toSymbol()”,query)
```
  out = []
  for term_idx of query
```
¶

console.log “k:”,term_idx
```
    term = query[term_idx]
```

console.log “ “, term_idx, term

    if term?
      for k,v of term
        out.push(k+":"+v)
  retval = out.join(',')

console.log(“ ==>”, retval)

  return retval

(exports ? this).NooDBAbstract = NooDBAbstract
(exports ? this).AbstractListener = AbstractListener
(exports ? this).SecureListener = SecureListener
(exports ? this).InsecureListener = InsecureListener
(exports ? this).Query = Query

noodbabstract.coffee

The user is in no groups, so can not be permitted because of membership.