noodb.coffee

Jump To … +

server.coffee src/actionknob.coffee src/autosem.coffee src/bitbucket_kba.coffee src/browserlog.coffee src/datareduction.coffee src/dci.coffee src/dciknob.coffee src/deeseeeye.coffee src/dnd.coffee src/doof.coffee src/formurla-mngr.coffee src/fractalpanel.coffee src/fractalpanel_test.coffee src/front.coffee src/ingestor.coffee src/kbabitbucket.coffee src/knobctrl.coffee src/lib_test.coffee src/nanoclock.coffee src/noodb.coffee src/noodbabstract.coffee src/noodbbrowser.coffee src/noodbbrowser_test.coffee src/noodbsec.coffee src/noorauth.coffee src/noorplugin.coffee src/noorquery.coffee src/noorvm.coffee src/noorwrite.coffee src/quadparser.coffee src/quadparsern3.coffee src/rbac.coffee src/reactor.coffee src/rebase.coffee src/rsrcidx.coffee src/sandboxactions.coffee src/screen_ctx.coffee src/spogi.coffee src/tabular_widget.coffee src/visctrl.coffee src/voicesknob.coffee src/whowhen.coffee src/xsd2native.coffee

noodb.coffee
¶

NooDB is a log-style graph database implemented in JS but ideally in C.

Replication, distribution and sharding shall be inspired by many sources: https://en.wikipedia.org/wiki/Syslog

crypto = require("crypto")
fs = require("fs")
path = require("path")
Log = require("log")
url = require("url")
N3 = require("n3")
N3Util = N3.Util

Clock = require("./nanoclock").Clock
date_to_now = require("./nanoclock").date_to_now
RsrcDb = require('./rsrcidx').RsrcDb
LineByLineReader = require("line-by-line")
parseQuadLine = require("./quadparser").parseQuadLine
parseQuadLineToQuint = require("./quadparser").parseQuadLineToQuint
Quad = require("./quadparser").Quad
Spogi = require("./spogi").Spogi
WhoWhen = require("./whowhen").WhoWhen
rebase = require("./rebase")
NooDBAbstract = require("./noodbabstract").NooDBAbstract
SecureListener = require("./noodbabstract").SecureListener
InsecureListener = require("./noodbabstract").InsecureListener
makeIngestor = require("./ingestor").makeIngestor
Ingestor = require("./ingestor").Ingestor

int_to_base = rebase.int_to_base
base_to_int = rebase.base_to_int
make_symbol = rebase.make_symbol

canonicalize_file_uri = (uri_or_literal, base) ->
  if N3Util.isLiteral(uri_or_literal)
    console.log("is #{uri_or_literal} a URI????")
    return uri_or_literal
  if uri_or_literal.indexOf(':') > -1
      a = 1
  return uri_or_literal

##’

class N5Parser
  constructor: (args) ->

  parse: (@lr, @noodb) ->
    @lr.on 'line', (line) =>
      @noodb.read_err__quint__prefix__(null, triple, null)
    @lr.on 'error', (err) =>
      @noodb.log.error(err)
    @lr.on 'end', () =>

class NooDB extends NooDBAbstract
  constructor: (@fname, @server_uri, args, read_callback) ->
    args = args or {}
    defaults =
      verbose: false
      warn_time_threshold: 5000
      verbosity: 3
      clock: new Clock()
      instrumented: false # instrumented = true turns on demonstrative code

What the default_graph should be is a puzzle, if it’s even needed. Should it be @server_uri? Or @server_uri + “/somepath/“?

      default_graph: "#{@server_uri}/erewhon/"
      last_user_no_int: Math.pow(57,3) - 1 # ie zzz in base57 so next_user_symbol is A222
      file_root: process.cwd()

    @resolve_defaults(defaults, args, Log) # TODO see if args.__proto__defaults would work
    rd_ctx =
      read_more: true
      file_spec_queue: []  # list of objects like {full_uri: '', full_path: ''o, raw_uri: '', is_dir: true} is_dir can be missing
      file_root: args.file_root

    @last_user_symbol = make_symbol(@last_user_no_int)
    @build_indices()
    @preload_prefixes()
    @name_it(read_callback, "outer")
    @init_db(rd_ctx, read_callback)

  universal_prefixes:
    nrn: 'http://nooron.com/_/'
    rdfs: 'http://www.w3.org/2000/01/rdf-schema#'
    rdf: 'http://www.w3.org/1999/02/22-rdf-syntax-ns#'
    xsd: 'http://www.w3.org/2001/XMLSchema#'
    naa: 'http://nooron.com/_/NooronAppArchitecture.ttl#'

  shutdown: (callback) ->
    if @writeable_fd?
      @log.notice "there is a @writeable_fd"
      fs.close @writeable_fd, () =>
        @log.notice "done closing #{@writeable_path}"
        callback()
    else
      @log.notice "there is a @writeable_fd"

  persist: (spogi) ->
    @log.info "persist():",spogi.i, spogi.constructor.name
    if @writeable_fd?

http://nodejs.org/api/fs.html#fs_fs_write_fd_buffer_offset_length_position_callback appends if file opened in a+ at current position SHOULD CHECK FOR OVERWRITING!!!

      if not spogi.asLine
        @log.debug spogi
        @log.debug "spogi.asLine is undefined, class:", spogi.constructor.name
        for k of spogi
          @log.error "  k:",k
      line = spogi.asN5Line() + "\n"  # was spogi.asLine()
      buffer = new Buffer(line)
      if buffer? and line.length > 9 # eg '<s> <p> 9'
        fs.writeSync(@writeable_fd, buffer, 0, buffer.length, null)

  get_array_of_random: (l) ->
    return crypto.randomBytes(l)

  server_log: (p, o) ->
    spogi = @allege(@server_uri, p, o, "nrn:server_log")
    @log.notice spogi + ""
    return spogi
  set_server_start_time: (date) ->
    @server_start_time = date

  open_db: (callback) ->
    @log.notice("SECURITY: q() should respect user constraint")
    writeables = @q(null, "rdf:type", "nrn:WriteableKB")
    writeable = writeables.last()
    if writeable?
      file_uri = writeable.s.key()
      if writeables.terms.length < 2
        @log.warning "open_db is using a dangerous workaround FIX the query problem"

file_path = file_uri.replace(/^file:\/\/\//, ‘’)

      file_path = @file_uri_to_fullpath(file_uri, process.cwd())
      if writeable.o.key() isnt "nrn:WriteableKB"
        throw new Error "QUERY problem: expecting 'nrn:WriteableKB', not \n" + file_uri
      @log.warning("SECURITY: fs.open paths should be sanitized and whitelisted")

http://nodejs.org/api/fs.html#fs_fs_open_path_flags_mode_callback

      if @writeable_fd
        throw new Error("@writeable_fd already defined")
      @writeable_fd = fs.openSync(file_path, "a+")
      @writeable_path = file_path

@server_log(“nrn:serverOpenedForWriting”, file_uri)

    else
      @log.notice "no writeable files found in " + writeables.all()
    callback()

  last_user_no: () ->
    make_symbol(@last_user_no_int)[0]

  next_user_symbol: () ->
    @last_user_no_int++
    int_symbol_pair = make_symbol(@last_user_no_int)
    @last_user_no_int = int_symbol_pair[1]
    @last_user_symbol = int_symbol_pair[0]
    return @last_user_symbol

  uri_resolve: (irreg_uri, cwd) ->
    return url.resolve(cwd, irreg_uri)

  file_uri_to_fullpath: (fileUri, cwd) ->

TODO harmonize with @resolve_file_path()

    if fileUri.startsWith('file:///')
      relPath = fileUri.replace(/^file:\/\//,"")
      return path.join(cwd, relPath)
    throw new Error("not a fileUri: #{fileUri}")

  uri_to_fullpath: (irreg_uri, cwd) ->

REVIEW this is a pointless POS which needs specification and testing

    reg_uri = irreg_uri.replace(/^file:/,"")
    if reg_uri == irreg_uri
      if irreg_uri[0] is '/'
        reg_uri = path.resolve(cwd, irreg_uri)
    console.error("uri_to_fullpath(#{irreg_uri}) ===> #{reg_uri}")
    return reg_uri

  get_stats: () ->
    return

  make_n3_assertion_handler: (basename, ext, default_graph, rd_ctx) ->
    return (error, quad, prefixes) =>

console.log “N3”, error, quad, prefixes

      if not quad # a null quad is sent to indicate end of stream
        console.log("  after reading",basename,ext, "read_count:", @read_count)
      if quad

the_graph = quad.graph isnt ‘’ and quad.graph or ‘nrn:’ + basename

        if quad.graph isnt '' and quad.graph
          the_graph = quad.graph
        else
          the_graph = default_graph

@warn_once(“graph is WRONGLY merely a qname #{the_graph}”)

        obj = quad.object
        if /ontology YAGO3/.exec(obj) and obj[0] isnt '"'
          @log.debug("======== n3_assertion_handler '...ontology YAGO3...' char1 is <#{obj[0]}> not '\"' ========")

¶

Strip the surrounding “” from string literals that N3 supplies obj = N3Util.isLiteral(obj) and N3Util.getLiteralValue(obj) or obj TODO(smurp): for some reason removing these breaks HuViz!!! http://localhost:9998/__/graph(g=nrn:foaf20140114) but removing them is needed by test: FNC can run functions which live in KBs on disk
```
        quint = [
          quad.subject,
          quad.predicate,
          obj,
          the_graph,
          @synthetic_key_factory_next()
        ]
```

console.log “N3”, error, quint, prefixes, basename console.log “N3”, prefixes, basename

        @read_count++

        if basename.includes('primordialSecurityKB') # or quint[0].includes('isPartOf') or true

@log.alert(“#{@read_count} #{quint.join(‘ ‘)}”)

          @log.alert("#{@read_count} basename=#{basename}")

@log.alert(“#{@read_count} basename=#{basename}”) # quint=[[#{quint}]] #{the_graph}”)

        false_or_spogi = @index_and_maybe_queue_for_reading(quint, rd_ctx)
        if false_or_spogi
          @log.alert("Oo.o".repeat(30)) # + " " + false_or_spogi.asLine())
        return false_or_spogi

  make_n3_prefix_handler: (basename, ext) ->
    return (prfx, rest) =>
      @log.debug("prefix_handler()", "@prefix #{prfx}: #{rest}")

  doof: ->
    if rd_ctx.read_more and (quint[2] in ['nrn:ReadableKB','nrn:WriteableKB'])
      full_path = @uri_resolve(quint[0], absdirname)
      rd_ctx.file_spec_queue.push(full_path)

rd_ctx.file_spec_queue.push(quint[0])

  queue_file_for_reading: (quint, rd_ctx) ->
    uri = quint[0]
    if uri.match(/(\~|.bak)$/)
      @log.debug("queue_file_for_reading() skipping uri: #{uri}")
      return
    file_spec =
      raw_uri: uri
    if uri.startsWith('file:')
      file_path = uri.substr(5) # trim off 'file:'
      if file_path.startsWith('///')
        file_path = file_path.substr(3)

rd_ctx.absdirname = path.dirname(path.resolve(file_path))

        file_spec.absdirname = path.dirname(path.resolve(file_path))
        try
          file_spec.full_path = @uri_resolve(quint[0], file_spec.absdirname)
        catch e
          @log.debug("rd_ctx",rd_ctx)
          throw e
      else
        throw new Error("URIs starting with 'file:' should start with 'file:///' -- #{quint[0]}")
    else if uri.indexOf(':') is -1 # ie it is not a url, hence it IS a relative path
      full_path = path.resolve(uri)
      canonical_file_path = full_path.replace(rd_ctx.file_root, 'file://')

@log.warning(“===================================================”) @log.warning(“full_path: #{full_path}”) @log.warning(“ uri: #{uri}”)

      @log.warning("canonical: #{canonical_file_path}")
      file_spec.full_uri = canonical_file_path
      file_spec.full_path = full_path
      if canonical_file_path.indexOf('.') is -1
        file_spec.is_dir = true # TODO base this on real test of disk
        @read_directory(file_spec.full_path, rd_ctx)

throw new Error(“uri: #{uri}, canonical_file_path: #{canonical_file_path} LOOKS LIKE A DIRECTORY”)

      quint[0] = canonical_file_path
    else
      @log.warning("queue_file_for_reading() ignoring #{quint[1]} #{quint[0]} because neither 'file:' nor relative path")
      return

    rd_ctx.file_spec_queue.push(file_spec)

  ext_to_mimetype:
    '.trig': 'application/trig'
    '.ttl': 'text/turtle'

  resolve_file_path: (uri, rd_ctx) ->

Purpose: if uri starts with ‘file:’ or is a relative uri return it

    @log.debug("resolve_file_path('#{uri}')\n") #, rd_ctx)
    if uri.endsWith('~')
      return false
    if uri.startsWith('file:///')
      nooron_root_relative_path = uri.replace(/^file:\/\/\//,'') # remove all three /
    if uri.indexOf(':') > -1 # bail because either uri uses a scheme other than file: or is a bad file path
      return
    if uri.startsWith('/')
      nooron_root_relative_path = uri.replace(/^\//, '')
    else
      nooron_root_relative_path = uri
    if nooron_root_relative_path
      full_path = path.resolve(rd_ctx.file_root, nooron_root_relative_path)
      @log.debug('return', full_path)
      return full_path
    @log.debug('return null')
    return # return nothing because uri was neither 'file:///' nor '/ROOT/RELATIVE/PATH' nor 'RELATIVE/PATH'
  read_db_n3: (fname, rd_ctx, callback) ->
    ext = path.extname(fname)
    format = @ext_to_mimetype[ext]
    @log.debug("read_db_n3(#{fname})")
    full_path = @resolve_file_path(fname, rd_ctx)
    @log.debug("resolve_file_path(#{fname}) ==> #{full_path}")
    if format and full_path
      @log.info("read_db_n3(#{fname})")
      parser = N3.Parser({format: format})

parser._setBase(‘file:///Users/smurp/REPOS/libnoo/‘)

      basename = path.basename(fname, ext)
      rdfStream = fs.createReadStream(fname)
      default_graph = 'nrn:' + basename

console.log format, ext

      parser.parse(rdfStream,
        @make_n3_assertion_handler(basename, ext, default_graph, rd_ctx),
        @make_n3_prefix_handler(basename, ext))
      return default_graph
    else
      return false

  ignore_file_or_dirname: (f_or_d) ->
    return f_or_d.endsWith('~') or
           f_or_d.endsWith('bak') or
           f_or_d is ('.DS_Store') or
           f_or_d is ('.git')

  read_db: (fname, rd_ctx, callback) ->
    super()

@old_read_db(fname, rd_ctx, callback)

    @bootstrap_ingestion(fname, rd_ctx, callback)

¶

########################### the Ingest/Consume process is meant as a rewrite of the read_db() system

###########################
```
  bootstrap_ingestion: (fullPath, rdCtx, callback) ->
```

This is the entrypoint for the ingestion process.

    stub =
      getRootPath: -> rdCtx.file_root
      noodb: @
      fullPath: rdCtx.file_root
    @rootIngestor = makeIngestor(fullPath, stub, callback, @, rdCtx)
    @rootIngestor.ingest()

  get_server_session: () ->

¶

The serverSession is the one which embodies the current execution of the server. Ideally it will have the session start time and the git code version associated with it.
```
    if not @session_no?
      all = @by_pred.getAll("nrn:serverStartedAt")
      @session_no = all.length + 1
```
¶

user_symbol this should be an identifier for the server

The user_symbol for a server should be the identifier for the server, generated by it’s proximate parent.
```
    retval = {user_symbol: 'B', session_no: @session_no} # TODO support multiple servers
```

@log.alert “get_server_session()”,JSON.stringify(retval)

    return retval

  subscribe_socket: (socket, qry) ->
    @subscribe_listener(new SocketListener(socket), qry)

class SocketListener extends InsecureListener
  constructor: (@socket) ->
    super()
  get_id: ->
    @socket.id
  send: (spogi) ->
    @socket.emit('from_upstream', spogi.forWire())
  getUser: (noodb) ->
    if @socket.user? # TODO @socket.rbac_session.getActor().getId()
      noodb.warn_once("SocketListener should extend SecureListener not InsecureListener")
      noodb.log.debug "  userId:", @socket.user.userId, "socket.id:", @socket.id
      return @socket.user
    else
      return

(exports ? this).NooDB = NooDB