Implement Nlp.js (#387)

* nlp

* split nodes

* entity node

* language drop down

* save and load model

* cleanup

* cleanup and language guess

* xleanup

* tests for entity

* test intent

* test nlp train and process

* cleanup

* update docs

* refactor language node

* language rule

* rule isLanguage

* lint

* fix text language

* update docs

CHANGELOG.md

@@ -1,3 +1,4 @@
+* **0.18.10** Added nodes for NLP.js, deprecated `Listen node`, breaking changes for `Language node`, added _Is Language ..._ rule to `Rules node` 
 * **0.18.9** Support for multiple webhook for Routee 
 * **0.18.8** Fix refresh of access token in Routee 
 * **0.18.7** Fixed Alexa nodes not properly working with the multi transport framework

__tests__/helpers.js

@@ -87,6 +87,35 @@ describe('Value extractors', () => {
     assert.equal(extractValue('boolean', 'aBoolean', RED.node , msg3), null);
   });
 
+  it('should extract an array of entities', () => {
+    const entities = [
+      { name: 'orange', aliases: ['orange blue', 'orange red'] },
+      { name: 'apple', aliases: null }
+    ];
+    const wrongentities = [
+      { name: 'orange', aliases: 'orange blue,orange red' },
+      { name: 'apple', aliases: null }
+    ];
+    const msg1 = RED.createMessage(entities);
+    const msg2 = RED.createMessage({ myentity: entities });
+    const msg3 = RED.createMessage(wrongentities);
+
+    const extract1 = extractValue('arrayOfEntities', 'entities', RED.node , msg1);
+    assert.isArray(extract1);
+    assert.lengthOf(extract1, 2);
+    assert.equal(extract1[0].name, 'orange');
+    assert.equal(extract1[1].name, 'apple');
+
+    const extract2 = extractValue('arrayOfEntities', 'myentity', RED.node , msg2);
+    assert.isArray(extract2);
+    assert.lengthOf(extract2, 2);
+    assert.equal(extract2[0].name, 'orange');
+    assert.equal(extract2[1].name, 'apple');
+
+    const extract3 = extractValue('arrayOfEntities', 'myentity', RED.node , msg3);
+    assert.isNull(extract3);
+  });
+
   it('should extract a string', () => {
     const msg1 = RED.createMessage('I am a string');
     const msg2 = RED.createMessage({ aString: 'I am a string' });

__tests__/language-node.js

@@ -1,72 +1,61 @@
-var _ = require('underscore');
-var assert = require('chai').assert;
-var RED = require('../lib/red-stub')();
-var LanguageBlock = require('../nodes/chatbot-language');
+const _ = require('underscore');
+const assert = require('chai').assert;
+const RED = require('../lib/red-stub')();
+const LanguageBlock = require('../nodes/chatbot-language');
 
-describe('Chat language node', function() {
+describe('Chat language node', () => {
 
-  it('should detect italian language', function() {
-    var msg = RED.createMessage({
-      content: 'hey guido come stai?',
+  it('should detect italian language and override it', () => {
+    const msg = RED.createMessage({
+      content: 'posso avere un gelato?',
       chatId: 42
     });
+    msg.chat().set('language', 'en');
     RED.node.config({
       language: 'italian',
       mode: 'medium'
     });
     LanguageBlock(RED);
     RED.node.get().emit('input', msg);
-    assert.equal(RED.node.message().payload.content, 'hey guido come stai?');
-    assert.equal(RED.node.message().payload.chatId, 42);
-    assert.equal(RED.node.message(1), null);
+    return RED.node.get().await()
+      .then(() => {
+        assert.equal(msg.chat().get('language'), 'it');
+        assert.equal(RED.node.message().payload.content, 'posso avere un gelato?');
+        assert.equal(RED.node.message().payload.chatId, 42);        
+      })    
   });
 
-  it('should not detect italian language', function() {
-    var msg = RED.createMessage({
-      content: 'hey guido come stai?',
-      chatId: 42
-    });
-    RED.node.config({
-      language: 'italian',
-      mode: 'strict'
-    });
-    LanguageBlock(RED);
-    RED.node.get().emit('input', msg);
-    assert.equal(RED.node.message(1).payload.content, 'hey guido come stai?');
-    assert.equal(RED.node.message(1).payload.chatId, 42);
-    assert.equal(RED.node.message(0), null);
-  });
-
-  it('should pass through commands', function() {
-    var msg = RED.createMessage({
+  it('should pass through commands', () => {
+    const msg = RED.createMessage({
       content: '/help',
       chatId: 42
     });
-    RED.node.config({
-      language: 'italian',
-      mode: 'strict'
-    });
+    msg.chat().set('language', 'it');
+    RED.node.config({});
     LanguageBlock(RED);
     RED.node.get().emit('input', msg);
-    assert.equal(RED.node.message(0).payload.content, '/help');
-    assert.equal(RED.node.message(0).payload.chatId, 42);
-    assert.equal(RED.node.message(1), null);
+    return RED.node.get().await()
+      .then(() => {
+        assert.equal(RED.node.message(0).payload.content, '/help');
+        assert.equal(RED.node.message(0).payload.chatId, 42);
+        assert.equal(msg.chat().get('language'), 'it');
+      });
   });
 
-  it('should detect english language', function() {
-    var msg = RED.createMessage({
-      content: 'how you doing?',
+  it('should detect english language', () => {
+    const msg = RED.createMessage({
+      content: 'What time is it?',
       chatId: 42
     });
-    RED.node.config({
-      language: 'english',
-      mode: 'medium'
-    });
+    RED.node.config({});
     LanguageBlock(RED);
     RED.node.get().emit('input', msg);
-    assert.equal(RED.node.message().payload.content, 'how you doing?');
-    assert.equal(RED.node.message().payload.chatId, 42);
-    assert.equal(RED.node.message(1), null);
+    return RED.node.get().await()
+      .then(() => {
+        assert.equal(msg.chat().get('language'), 'en');
+        assert.equal(RED.node.message().payload.content, 'What time is it?');
+        assert.equal(RED.node.message().payload.chatId, 42);
+      });
   });
 
 });

__tests__/nlp-entity-node.js

@@ -0,0 +1,110 @@
+const _ = require('underscore');
+const assert = require('chai').assert;
+const RED = require('../lib/red-stub')();
+const NLPEntityBlock = require('../nodes/chatbot-nlp-entity');
+
+require('../lib/platforms/telegram');
+require('../lib/platforms/slack');
+
+describe('Chat nlp entity node', () => {
+
+  it('set the entity payload', () => {
+    const msg = RED.createMessage();
+    RED.node.config({
+      name: 'fruits',
+      language: 'en',
+      entities: [
+        { name: 'orange' },
+        { name: 'apple', aliases: ['golden apple', 'granny smith apple'] } 
+      ]
+    });
+    NLPEntityBlock(RED);
+    RED.node.get().emit('input', msg);
+    return RED.node.get().await()
+      .then(() => {
+        const payload = RED.node.message().payload;
+        assert.isObject(payload);
+        assert.isObject(payload.entities);
+        assert.isObject(payload.entities.en);
+        assert.isArray(payload.entities.en.fruits);
+        assert.lengthOf(payload.entities.en.fruits, 2);
+        assert.equal(payload.entities.en.fruits[0].name, 'orange');
+        assert.equal(payload.entities.en.fruits[1].name, 'apple');
+        assert.isArray(payload.entities.en.fruits[1].aliases);
+        assert.lengthOf(payload.entities.en.fruits[1].aliases, 2);
+        assert.equal(payload.entities.en.fruits[1].aliases[0], 'golden apple');
+        assert.equal(payload.entities.en.fruits[1].aliases[1], 'granny smith apple');
+      });
+
+  });
+
+  it('set the entity payload from message', () => {
+    const msg = RED.createMessage({
+      name: 'fruits',
+      language: 'en',
+      entities: [
+        { name: 'orange' },
+        { name: 'apple', aliases: ['golden apple', 'granny smith apple'] } 
+      ]
+    });
+    RED.node.config({});
+    NLPEntityBlock(RED);
+    RED.node.get().emit('input', msg);
+    return RED.node.get().await()
+      .then(() => {
+        const payload = RED.node.message().payload;
+        assert.isObject(payload);
+        assert.isObject(payload.entities);
+        assert.isObject(payload.entities.en);
+        assert.isArray(payload.entities.en.fruits);
+        assert.lengthOf(payload.entities.en.fruits, 2);
+        assert.equal(payload.entities.en.fruits[0].name, 'orange');
+        assert.equal(payload.entities.en.fruits[1].name, 'apple');
+        assert.isArray(payload.entities.en.fruits[1].aliases);
+        assert.lengthOf(payload.entities.en.fruits[1].aliases, 2);
+        assert.equal(payload.entities.en.fruits[1].aliases[0], 'golden apple');
+        assert.equal(payload.entities.en.fruits[1].aliases[1], 'granny smith apple');
+      });
+
+  });
+
+  it('set the entity payload from message and append to previous', () => {
+    const msg = RED.createMessage({
+      entities: { 
+        en: { 
+          fruits: [
+            { name: 'banana' }
+          ] 
+        } 
+      }
+    });
+    RED.node.config({
+      name: 'fruits',
+      language: 'en',
+      entities: [
+        { name: 'orange' },
+        { name: 'apple', aliases: ['golden apple', 'granny smith apple'] } 
+      ]
+    });
+    NLPEntityBlock(RED);
+    RED.node.get().emit('input', msg);
+    return RED.node.get().await()
+      .then(() => {
+        const payload = RED.node.message().payload;
+        assert.isObject(payload);
+        assert.isObject(payload.entities);
+        assert.isObject(payload.entities.en);
+        assert.isArray(payload.entities.en.fruits);
+        assert.lengthOf(payload.entities.en.fruits, 3);
+        assert.equal(payload.entities.en.fruits[0].name, 'banana');
+        assert.equal(payload.entities.en.fruits[1].name, 'orange');
+        assert.equal(payload.entities.en.fruits[2].name, 'apple');
+        assert.isArray(payload.entities.en.fruits[2].aliases);
+        assert.lengthOf(payload.entities.en.fruits[2].aliases, 2);
+        assert.equal(payload.entities.en.fruits[2].aliases[0], 'golden apple');
+        assert.equal(payload.entities.en.fruits[2].aliases[1], 'granny smith apple');
+      });
+  });
+
+});
+

__tests__/nlp-intent-node.js

@@ -0,0 +1,101 @@
+const _ = require('underscore');
+const assert = require('chai').assert;
+const RED = require('../lib/red-stub')();
+const NLPEntityBlock = require('../nodes/chatbot-nlp-intent');
+
+require('../lib/platforms/telegram');
+require('../lib/platforms/slack');
+
+describe('Chat nlp intent node', () => {
+
+  it('set the intent payload', () => {
+    const msg = RED.createMessage();
+    RED.node.config({
+      intent: 'switch.on',
+      language: 'en',
+      utterances: [
+        'switch on the lights',
+        'turn on the lights' 
+      ]
+    });
+    NLPEntityBlock(RED);
+    RED.node.get().emit('input', msg);
+    return RED.node.get().await()
+      .then(() => {
+        const payload = RED.node.message().payload;
+        assert.isObject(payload);
+        assert.isObject(payload.intents);
+        assert.isObject(payload.intents.en);
+        assert.isArray(payload.intents.en['switch.on']);
+        assert.lengthOf(payload.intents.en['switch.on'], 2);
+        assert.equal(payload.intents.en['switch.on'][0], 'switch on the lights');
+        assert.equal(payload.intents.en['switch.on'][1], 'turn on the lights');
+
+      });
+  });
+
+  it('set the intent payload from message', () => {
+    const msg = RED.createMessage({
+      intent: 'switch.on',
+      language: 'en',
+      utterances: [
+        'switch on the lights',
+        'turn on the lights' 
+      ]
+    });
+    RED.node.config({});
+    NLPEntityBlock(RED);
+    RED.node.get().emit('input', msg);
+    return RED.node.get().await()
+      .then(() => {
+        const payload = RED.node.message().payload;
+        assert.isObject(payload);
+        assert.isObject(payload.intents);
+        assert.isObject(payload.intents.en);
+        assert.isArray(payload.intents.en['switch.on']);
+        assert.lengthOf(payload.intents.en['switch.on'], 2);
+        assert.equal(payload.intents.en['switch.on'][0], 'switch on the lights');
+        assert.equal(payload.intents.en['switch.on'][1], 'turn on the lights');
+
+      });
+  });
+
+  it('set the intent payload and append to previous', () => {
+    const msg = RED.createMessage({
+      intents: { 
+        en: { 
+          'switch.on': ['switch on all the lights'] 
+        },
+        it: { 
+          'switch.on': ['appiccia la luce'] 
+        } 
+      } 
+    });
+    RED.node.config({
+      intent: 'switch.on',
+      language: 'en',
+      utterances: [
+        'switch on the lights',
+        'turn on the lights' 
+      ]
+    });
+    NLPEntityBlock(RED);
+    RED.node.get().emit('input', msg);
+    return RED.node.get().await()
+      .then(() => {
+        const payload = RED.node.message().payload;      
+        assert.isObject(payload);
+        assert.isObject(payload.intents);
+        assert.isObject(payload.intents.en);
+        assert.isArray(payload.intents.en['switch.on']);
+        assert.lengthOf(payload.intents.en['switch.on'], 3);
+        assert.equal(payload.intents.en['switch.on'][0], 'switch on all the lights');
+        assert.equal(payload.intents.en['switch.on'][1], 'switch on the lights');
+        assert.equal(payload.intents.en['switch.on'][2], 'turn on the lights');
+        assert.isArray(payload.intents.it['switch.on']);
+        assert.lengthOf(payload.intents.it['switch.on'], 1);
+        assert.equal(payload.intents.it['switch.on'][0], 'appiccia la luce');
+      });
+  });
+});
+