Skip to content
This repository has been archived by the owner on Mar 1, 2021. It is now read-only.

Commit

Permalink
mantap tabelnya bener, makasih alvin dan alex
Browse files Browse the repository at this point in the history
  • Loading branch information
Josep Marcello committed Nov 15, 2020
1 parent 6af7d19 commit 50545fc
Show file tree
Hide file tree
Showing 9 changed files with 255 additions and 81 deletions.
27 changes: 17 additions & 10 deletions src/backend/main.js
Original file line number Diff line number Diff line change
@@ -1,20 +1,20 @@
'use strict';

/**
* Imports
*/
/* *** Imports *** */
// from npm library
const express = require('express');
const multer = require('multer');
const path = require('path');
const fs = require('fs');
const bp = require('body-parser');

// from other js files in this project
const proc = require('./process');
const parsedoc = require('./parsedoc');
const scraper = require('./scraper');

const GFileDir = './src/frontend/public/uploads/';

/* *** Global variables *** */
const GFileDir = './src/frontend/public/uploads/'; // path ke tempat upload
let GFilesList = []; // List files objects

// konfigurasi server express baru
Expand All @@ -26,12 +26,13 @@ const serverConfig = {
// Inisialisasi instance express baru
const app = express();

// setup middleware untuk express
app.use(express.static(path.join(__dirname, '../frontend/')));
app.use(bp.urlencoded({ extended: false }));
app.use(bp.json());

/**
* Routing untuk memberikan query search
* Routing ke API searching
*/
app.get('/search', (req, res) => {
const query = req.query.q.toLowerCase();
Expand All @@ -44,10 +45,9 @@ app.get('/search', (req, res) => {
});

/**
* Routing untuk menguji searching
* Routing ke API pengujian searching
*/
app.get('/test', (req, res) => {
//GFilesList = [];
const query = req.query.q;

const hasil = proc.testProcess(query);
Expand All @@ -56,6 +56,9 @@ app.get('/test', (req, res) => {
return res.json(hasil);
});

/**
* Routing ke API web scaper
*/
app.post('/scraper', async (req, res) => {
// hayolo tiba2 angkatan 19 dipanggil
const result = await scraper.extractHTML(
Expand All @@ -72,9 +75,13 @@ app.post('/scraper', async (req, res) => {
* Routing ke start-page
*/
app.get('/', (req, res) => {
res.sendFile(path.join(__dirname, '../frontend/public/index.html'));
//res.sendFile(path.join(__dirname, '../frontend/public/index.html'));
res.send('Hi');
});

/**
* filter file untuk multer
*/
const filterfile = function (req, file, cb) {
// Accept txt or html only
if (!file.originalname.match(/(txt|html)$/i)) {
Expand Down Expand Up @@ -103,7 +110,7 @@ const storage = multer.diskStorage({
});

/**
* Routing untuk nerima upload
* Routing ke API upload file
*/
let upload = multer({ storage: storage, fileFilter: filterfile }).array(
'files',
Expand Down
64 changes: 43 additions & 21 deletions src/backend/process.js
Original file line number Diff line number Diff line change
@@ -1,24 +1,28 @@
/* *** Bentuk object untuk document *** */
/* doc = {
* filename : Val, // dari awal
* konten : Val, // dari awal
* kontenOriginal: Val, // dari awal
* vektor : Val, // ditambah di tengah
* similarity : Val, // ditambah di tengah
* firstSentence : Val, // ditambah di tengah
* filename : Val, // dari awal, nama file
* konten : Val, // dari awal, isi dokumen yang udah di-stem dan diapus stopwords-nya
* kontenOriginal: Val, // dari awal, isi dokumen asli
* vektor : Val, // ditambah di tengah, vektor dari dokumen
* similarity : Val, // ditambah di tengah, hasil cosine similarity
* firstSentence : Val, // ditambah di tengah, kalimat pertama tiap dokumen
* },
*
* untuk menyimpan kata-kata pada query, akan digunakan suatu list of
* strings
* queryWords = [Q1, Q2, Q3, ...]
*/

/* *** Imports *** */
// npm library
const sastrawi = require('sastrawijs');
const stopwords = require('./stopwords');
const parsedoc = require('./parsedoc');

// other js files in this project
const stopwordsID = stopwords.stopwordsID;

/* *** global variables *** */
let termDictionary = [];

/**
Expand Down Expand Up @@ -62,6 +66,7 @@ const removeStopwords = (string) => {
}
}

// Masukin kata yg buka stopwrod ke hasil
for (let i = 0; i < kata.length; i++) {
let katabersih = kata[i].split('.').join('');
if (!stopwordsID.includes(katabersih) || count == 0) {
Expand All @@ -80,7 +85,7 @@ const removeStopwords = (string) => {
const cleanString = (str) => {
str = stemString(str);
str = removeStopwords(str);
str.trim();
str.trim(); // Biar ga ada spasi di blkg atau di depan

return str;
};
Expand All @@ -92,15 +97,17 @@ const cleanString = (str) => {
* @Returns {String} str yang sudah dibersihkan
*/
const removeEscapeChr = (str) => {
const escapeChrRE = /(\n|\r|\t)/g;
const whitespaceRE = /\s\s+/g;
const escapeChrRE = /(\n|\r|\t)/g; // RE untuk karakter \n, \t, \r
const whitespaceRE = /\s\s+/g; // RE untuk spasi yg muncul lebih dari satu kali
str = str.replace(escapeChrRE, ' ');
str = str.replace(whitespaceRE, ' ');
return str;
};

/**
* TODO: comment
* Fungsi untuk menambahkan kata-kata pada str ke dalam database (dalam hal
* ini, adalah variabel global termDictionary)
* @param {String} str - string yang kata-katanya ingin ditambahkan ke databse
*/
const addToTermDict = (str) => {
str = str.split(' ');
Expand Down Expand Up @@ -142,6 +149,8 @@ const createDocQueryObj = (docContent, wordList) => {
const retObj = {};

for (const key of wordList) {
// kalau 'key' ada di dokumen, maka banyaknya akan disalin ke dalam object,
// jika tidak maka akan disalin nilai 0
retObj[key] = key in docObj ? docObj[key] : 0;
}

Expand All @@ -165,21 +174,25 @@ const sortSimilaritiesDsc = (arrObj) => {
* @Returns String kalimat pertama dari str
*/
const makeFirstSentence = (str) => {
const pemisah = '.';
const period = '.';
const charCount = 100;

let existsPeriod = false;
let i = 0;
for (; i < str.length && !existsPeriod; ++i) {
const chr = str[i];
if (chr === pemisah) {
if (chr === period) {
existsPeriod = true;
}
}

// Banyak karakter tidak sampai charCount dan tidak ada period
if (str.length <= charCount && !existsPeriod) return str;
// banyak karakter melebihi charCount atau ada priod
else if (!existsPeriod || i > charCount)
// potong lalu tambahkan elipsis
return `${str.slice(0, charCount)}...`;
// ada period dan banyak karakter kurang dari charCount
else return str.slice(0, i);
};

Expand Down Expand Up @@ -210,13 +223,16 @@ const cosineSim = (Q, D) => {
let mQ = 0; // ||Q||
let mD = 0; // ||D||

// mencari hasil penjumlahan kuadrat tiap elemen di Q dan D
for (i = 0; i < Q.length; i++) {
mQ += Q[i] * Q[i];
mD += D[i] * D[i];
}

// menghitung panjang vetkor Q dan D
mQ = Math.sqrt(mQ);
mD = Math.sqrt(mD);

//console.log({ dotproduct: dotproduct, mQ: mQ, mD: mD });
return dotproduct / (mQ * mD); // rumus cosine sim
};
Expand All @@ -234,7 +250,11 @@ const toVector = (obj) => {
};

/**
* TODO: comment
* Fungsi untuk memeriksa apakah di suatu list dokumen sudah memiliki dokumen
* dengan **nama** yang sama
* @param {Array} list - list dokumen yang ingin diperiksa
* @param {Object} obj - object file yang ingin diperiksa
* @Returns suatu nilai Boolean, true jika obj ada di list, false jika tidak
*/
exports.containsFile = (list, obj) => {
for (let i = 0; i < list.length; ++i) {
Expand All @@ -251,6 +271,7 @@ exports.containsFile = (list, obj) => {
/**
* Fungsi proses yang digunakan untuk menguji algoritma pencarian
* @params {string} query - query search
* TODO: Comemnts
*/
exports.testProcess = (query) => {
/* *** SETUP UNTUK TESTING *** */
Expand Down Expand Up @@ -281,10 +302,10 @@ exports.testProcess = (query) => {

console.log(docs);
const queryObj = {
query: query,
cleanQuery: cleanQuery,
queryWords: queryWords,
vetor: queryVec,
query: query, // query original (yg dikirim user)
cleanQuery: cleanQuery, // query yg udh dibersihin (udh di-stem, diapusin stopwords)
queryWords: queryWords, // list of kata-kata query yang udh dibersihin
vector: queryVec, // jumlah kemunculan tiap kata pada query yg koresponden dengan kamus kata
};
return { docs: docs, query: queryObj };
};
Expand All @@ -293,6 +314,7 @@ exports.testProcess = (query) => {
* Fungsi proses yang digunakan untuk menguji algoritma pencarian
* @params {string} query - query search
* @params {object[]} docs - berisi object dokumen
* TODO: Comemnts
*/
exports.mainProcess = (query, docs) => {
termDictionary = [];
Expand All @@ -319,10 +341,10 @@ exports.mainProcess = (query, docs) => {

console.log(docs);
const queryObj = {
query: query,
cleanQuery: cleanQuery,
queryWords: queryWords,
vector: queryVec,
query: query, // query original (yg dikirim user)
cleanQuery: cleanQuery, // query yg udh dibersihin (udh di-stem, diapusin stopwords)
queryWords: queryWords, // list of kata-kata query yang udh dibersihin
vector: queryVec, // jumlah kemunculan tiap kata pada query yg koresponden dengan kamus kata
};

return { docs: docs, query: queryObj };
Expand Down
32 changes: 21 additions & 11 deletions src/frontend/src/App.vue
Original file line number Diff line number Diff line change
@@ -1,14 +1,16 @@
<template>
<div class="section">
<div class="container">
<h1 style = " font-size: 100%;">Search It</h1>
<button @click="showSearch" class="button is-info">Search</button>
<button @click="showScraper" class="button is-info">Webscrape</button>
<button @click="showAbout" class="button is-info">Perihal</button>
<uploadedFile v-if="search"/>
<search v-if="search" />
<webScrape v-if="scraper"/>
<aboutUs v-if="about"/>
<div class="background">
<div class="section">
<div class="container">
<h1 style = " font-size: 500%; font-family: 'Open Sans', sans-serif; padding-bottom:15px">Search-It</h1>
<button @click="showSearch" class="button is-info is-success">Search</button>
<button @click="showScraper" class="button is-info is-success">Webscrape</button>
<button @click="showAbout" class="button is-info is-success">Perihal</button>
<uploadedFile v-if="search"/>
<search v-if="search" />
<webScrape v-if="scraper"/>
<aboutUs v-if="about"/>
</div>
</div>
</div>
</template>
Expand Down Expand Up @@ -63,6 +65,14 @@ export default {
text-align: center;
color: #2c3e50;
margin-top: 60px;
}
.background{
background-color:rgb(248, 194, 145);
}
.button{
margin:10px 10px 20px 10px;
}
.button.is-success:hover{
background-color: black;
}
</style>
4 changes: 2 additions & 2 deletions src/frontend/src/components/aboutUs.vue
Original file line number Diff line number Diff line change
Expand Up @@ -51,8 +51,8 @@ export default {
text-align: initial;
}
.middle{
margin: 100px;
padding: 50px;
margin: 0px 100px 100px 100px;
padding: 0px 50px 50px 50px;
justify-content: center;
}
p{
Expand Down
28 changes: 20 additions & 8 deletions src/frontend/src/components/search.vue
Original file line number Diff line number Diff line change
@@ -1,12 +1,15 @@
<template>
<div class="search-bar">
<form action="/" method="get" v-on:submit.prevent="submitQuery">
<input type="text" placeholder="Type query here..." id="q" name="q" v-model="searchStr"/>
<!-- Kalo bisa pake nerd font aja biar jadi ikon search itu -->
<button type="submit">
<i class ="fas fa-search"></i>
</button>

<div class ="field" style="width:inherit; height:inherit">
<input class="input" type="text" style="width:60%" placeholder="Type query here..." id="q" name="q" v-model="searchStr"/>
<button type="submit" style="width:inherit; height:inherit">
<span class="icon is-medium">
<i class ="fas fa-search"></i>
</span>
</button>
<!-- Kalo bisa pake nerd font aja biar jadi ikon search itu -->
</div>
</form>
</div>

Expand All @@ -21,13 +24,19 @@
</div>
</div>

<table>
</table>
<tableQuery :isSearched="this.isSearched" :docs="this.docs" :query="this.queryResult"/>

</template>

<script>
import tableQuery from './tableQuery.vue';
export default {
name: 'search',
components: {
tableQuery,
},
data() {
return {
searchStr: '',
Expand Down Expand Up @@ -83,4 +92,7 @@ export default {
text-align: left;
padding: 20px 0px;
}
.input{
opacity: 70%;
}
</style>
Empty file.
Loading

0 comments on commit 50545fc

Please sign in to comment.