spec-part2.ctr

Broom memoryLimit: 1024 * 1024 * 1024 * 8.

import main: \*.
import
  Library/Test: 'describe'
.

#:language XFrozen

describe TokenAnalyser do: {:it:let
  it exists do: {
    { TokenAnalyser. } should not raiseError: String.
  }.

  let[Reflect thisContext] lexer { Lexer }.
  let[Reflect thisContext] analyser { NGramHelper }.
  let[Reflect thisContext] glblctx { Reflect thisContext }.
  let[Reflect thisContext] tokenmap { File new: 'tokenMap', open: 'r' }.
  let[Reflect thisContext] limit { 5000 }.
  let[Reflect thisContext] ngram-N { 3 }.

  var fancy-name is {\:ex
    (frozen _ is Number
      case: 1 do: {\ 'uni'},
      case: 2 do: {\ 'bi'},
      case: 3 do: {\ 'tri'},
      case: 4 do: {\ 'quadri'},
      default:    {\ 'insane'}) switch: ex.
  }.

  it can tokenise and calculate NGram do: {:thisTest
    var allTargetTexts is (Generator from: 2 to: limit) fmap: (\:x File new: 'samples/sample$$x', read).

    lexer collate: (File new: 'collation', read split: '\n', fmap: {:x ^x split: ' ' max: 1. }).

    thisTest tests: '${{fancy-name[ngram-N]}}$gram'.
    thisTest progressesUpTo: limit - ngram-N + 1.
    var res is analyser withN: ngram-N,
      ngramCountOnTexts: allTargetTexts
      withLexer: lexer
      pre: {:i thisTest progress: i.}
      post: {:i thisTest show.}.

    var tokenTable is res last.
    var ngramCount is res head.
    var ngramCountVec is ngramCount extractDimension: ngram-N - 1 atIndex: 0, multiplicativeInverse.
    ngramCountVec writeInto: 'countinv'.
    var ngramProb is ngramCount multiply: ngramCountVec throughDimension: ngram-N - 1.

    ngramCount writeInto: 'count'.
    ngramProb writeInto: 'prob'.
    tokenTable each: {:k:v
      tokenmap write: '$$k : $$v \n'.
    }.
    tokenmap close.
  }.

}.