dataset view

magdabot · Mar 16, 2021 · dbdafdc · dbdafdc
1 parent 6c32e8e
commit dbdafdc
Show file tree

Hide file tree

Showing 9 changed files with 110 additions and 97 deletions.
diff --git a/magda-typescript-common/package.json b/magda-typescript-common/package.json
@@ -22,6 +22,7 @@
     "@types/chai": "^4.0.0",
     "@types/chai-as-promised": "^7.1.0",
     "@types/cross-spawn": "^6.0.1",
+    "@types/dompurify": "^2.2.1",
     "@types/express": "^4.17.6",
     "@types/lodash": "^4.14.73",
     "@types/mocha": "^2.2.41",
@@ -50,20 +51,20 @@
     "@types/resolve": "^1.14.0",
     "@types/vfile-message": "^2.0.0",
     "body-parser": "^1.18.2",
+    "dompurify": "^2.0.3",
     "express": "^4.17.1",
     "isomorphic-fetch": "^2.2.1",
     "jsonwebtoken": "^8.4.0",
     "load-json-file": "^1.0.0",
     "lodash": "^4.17.11",
+    "markdown-it": "^10.0.0",
     "moment": "^2.17.1",
     "read-pkg-up": "^3.0.0",
     "request": "^2.88.0",
     "resolve": "^1.15.0",
     "tsmonad": "^0.7.2",
     "urijs": "^1.19.4",
     "uuid": "^8.2.0",
-    "markdown-it": "^10.0.0",
-    "dompurify": "^2.0.3",
     "yargs": "^12.0.5"
   },
   "config": {

diff --git a/magda-typescript-common/src/markdownToHtml.ts b/magda-typescript-common/src/markdownToHtml.ts
@@ -1,5 +1,5 @@
 import MarkdownIt from "markdown-it";
-const DOMPurify = require("dompurify/dist/purify");
+import DOMPurify from "dompurify";
 
 const md = new MarkdownIt({
     html: true,

diff --git a/magda-web-server/src/cralwerViews/commonView.ts b/magda-web-server/src/cralwerViews/commonView.ts
@@ -18,7 +18,7 @@ const commonView = (
             <style></style>
         </head>
         <body>
-            ${markdownToHtml(__content)}
+            ${markdownToHtml(__content, true)}
             ${
                 shouldShowFullVersionLink && fullVersionUrl
                     ? `<br />

diff --git a/magda-web-server/src/cralwerViews/dataset.ts b/magda-web-server/src/cralwerViews/dataset.ts
@@ -26,7 +26,7 @@ const dataset = (record: Record, baseUrl: string) => {
         : [];
 
     return `---
-title: Dataset: ${title}
+title: ${JSON.stringify(`Dataset: ${title}`)}
 ---
 # Dataset: ${title}
 ---

diff --git a/magda-web-server/src/index.ts b/magda-web-server/src/index.ts
@@ -362,8 +362,11 @@ app.get(["/", "/index.html*"], async function (req, res) {
 // app.use("/admin", express.static(adminBuild));
 app.use(express.static(clientBuild));
 
+console.log("enableCrawlerViews: ", argv.enableCrawlerViews);
+console.log("enableDiscourseSupport: ", argv.enableDiscourseSupport);
+
 // crawler view router
-if (argv.enableCrawlerViews) {
+if (argv.enableCrawlerViews || argv.enableDiscourseSupport) {
     app.use(
         createCralwerViewRouter({
             registryApiBaseUrl: argv.registryApiBaseUrlInternal,

diff --git a/magda-web-server/src/test/cralwerViews/dataset.spec.ts b/magda-web-server/src/test/cralwerViews/dataset.spec.ts
@@ -12,7 +12,7 @@ describe("Test cralwer view: dataset", () => {
             "https://test.com/"
         ).replace(/\r\n/g, "\n");
         const targetContent = fse
-            .readFileSync(path.resolve(__dirname, "./sampleDatasetView.md"), {
+            .readFileSync(path.resolve(__dirname, "./sampleDatasetView.txt"), {
                 encoding: "utf-8"
             })
             .replace(/\r\n/g, "\n");

diff --git a/magda-web-server/src/test/cralwerViews/sampleDatasetView.md b/magda-web-server/src/test/cralwerViews/sampleDatasetView.md
diff --git a/magda-web-server/src/test/cralwerViews/sampleDatasetView.txt b/magda-web-server/src/test/cralwerViews/sampleDatasetView.txt
@@ -0,0 +1,87 @@
+---
+title: "Dataset: ASIC – Credit Licensee Dataset"
+---
+# Dataset: ASIC – Credit Licensee Dataset
+---
+## General Information
+
+- Australian Securities and Investments Commission (ASIC) (Commonwealth of Australia)
+- Create Date: 22/12/2015
+- Update Date: 01/02/2021
+- Landing Page: [https://data.gov.au/dataset/fa0b0d71-b8b8-4af8-bc59-0b000ce0d5e4](https://data.gov.au/dataset/fa0b0d71-b8b8-4af8-bc59-0b000ce0d5e4)
+- Keywords:
+  - ASIC
+  - Credit
+  - Credit Licence
+  - Credit Licensee
+  - Licence
+  - Licencee
+  - Register
+  - australian government
+  - federal government
+
+## Description
+
+###Update April 2019 - addition of license authorisations to Credit Licensee dataset ###
+
+From 4 April 2019, the Credit Licensee dataset will include license authorisations. See the updated helpfile for further information.
+
+***
+
+ASIC is Australia’s corporate, markets and financial services regulator. ASIC contributes to Australia’s economic reputation and wellbeing by ensuring that Australia's financial markets are fair and transparent, and supported by confident and informed investors and consumers. 
+
+Australian Credit Licensees are required to maintain their details on ASIC's registers. Information contained on the Credit Licensee Register is made available to the public to search via the ASIC Connect website. 
+
+Selected data from the register will be uploaded each month to www.data.gov.au. The data made available will be a snapshot of the register at a point in time. Legislation prescribes the type of information ASIC is allowed to disclose to the public. 
+
+The information in the downloadable dataset includes: 
+
+* Register Name
+* Credit licensee number
+* Credit licensee name
+* Date licence was issued
+* Date licence was ceased (if applicable)
+* Credit licensee status
+* Credit licensee ABN or ACN(if applicable)
+* AFSL number
+* Credit licence status history
+* Principal business locality
+* Principal business state/territory
+* Principal business postcode
+* Latitude coordinates
+* Longitude coordinates
+* EDRS code
+* Business name (if applicable)
+* License authorisations
+
+Additional information about Credit Licensees can be found via ASIC's website (http://www.asic.gov.au). To view some information you may be charged a fee.
+
+More information about searching ASIC's registers (http://www.asic.gov.au/online-services/search-asics-registers/).
+
+## Distributions
+
+- [Credit Licence Dataset - Current](https://test.com//dataset/ds-dga-fa0b0d71-b8b8-4af8-bc59-0b000ce0d5e4/distribution/dist-dga-35953a01-a9a8-4609-8566-c9fa7de465d3)
+  - Format: CSV
+  - License: Creative Commons Attribution 3.0 Australia
+  - Create Date: 22/12/2015
+  - Update Date: 01/02/2021
+- [Credit Licensee - Help File](https://test.com//dataset/ds-dga-fa0b0d71-b8b8-4af8-bc59-0b000ce0d5e4/distribution/dist-dga-48c6cd71-2339-41e7-bc93-34786edf3cd8)
+  - Format: PDF
+  - License: Creative Commons Attribution 3.0 Australia
+  - Create Date: 23/12/2015
+  - Update Date: 04/04/2019
+- [Credit Licence Dataset - National Map](https://test.com//dataset/ds-dga-fa0b0d71-b8b8-4af8-bc59-0b000ce0d5e4/distribution/dist-dga-548075a1-c36e-4837-bf26-cc00567b5c23)
+  - Format: CSV-GEO-AU
+  - License: Creative Commons Attribution 3.0 Australia
+  - Create Date: 24/12/2015
+  - Update Date: 01/02/2021
+- [Credit Licence Dataset - Current](https://test.com//dataset/ds-dga-fa0b0d71-b8b8-4af8-bc59-0b000ce0d5e4/distribution/dist-dga-546d3dae-13db-4c1d-bfb3-b5a9d0cbbea1)
+  - Format: XLSX
+  - License: Creative Commons Attribution 3.0 Australia
+  - Create Date: 07/04/2016
+  - Update Date: 01/02/2021
+- [Credit Licence Dataset - Current](https://test.com//dataset/ds-dga-fa0b0d71-b8b8-4af8-bc59-0b000ce0d5e4/distribution/dist-dga-3abf1383-c4e5-4c1a-8331-8434b17b6f10)
+  - Format: TSV
+  - License: Creative Commons Attribution 3.0 Australia
+  - Create Date: 09/11/2016
+  - Update Date: 01/02/2021
diff --git a/yarn.lock b/yarn.lock
@@ -3101,6 +3101,13 @@
   resolved "https://registry.yarnpkg.com/@types/debounce-promise/-/debounce-promise-3.1.2.tgz#e28b8578e859d1943fe1850ce300c6acda1f7af3"
   integrity sha512-MRQEy2LooG621ln0JOQXjcVIZWWY2VkefIxeO1cpO57tCnYAQdmq+b7CMw7pQC6RCc1itfcdwJKdF9/y4cLoZA==
 
+"@types/dompurify@^2.2.1":
+  version "2.2.1"
+  resolved "https://registry.yarnpkg.com/@types/dompurify/-/dompurify-2.2.1.tgz#eebf3af8afe2f577a53acab9d98a3a4cb04bbbe7"
+  integrity sha512-3JwbEeRVQ3n6+JgBW/hCdkydRk9/vWT+UEglcXEJqLJEcUganDH37zlfLznxPKTZZfDqA9K229l1qN458ubcOQ==
+  dependencies:
+    "@types/trusted-types" "*"
+
 "@types/email-validator@^1.0.6":
   version "1.0.6"
   resolved "https://registry.yarnpkg.com/@types/email-validator/-/email-validator-1.0.6.tgz#d555e53c826ac75e0205305f33ff47fc46930cd3"
@@ -3687,6 +3694,11 @@
   resolved "https://registry.yarnpkg.com/@types/tough-cookie/-/tough-cookie-4.0.0.tgz#fef1904e4668b6e5ecee60c52cc6a078ffa6697d"
   integrity sha512-I99sngh224D0M7XgW1s120zxCt3VYQ3IQsuw3P3jbq5GG4yc79+ZjyKznyOGIQrflfylLgcfekeZW/vk0yng6A==
 
+"@types/trusted-types@*":
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/@types/trusted-types/-/trusted-types-2.0.0.tgz#aee6e868fcef74f2b8c71614b6df81a601a42f17"
+  integrity sha512-I8MnZqNXsOLHsU111oHbn3khtvKMi5Bn4qVFsIWSJcCP1KKDiXX5AEw8UPk0nSopeC+Hvxt6yAy1/a5PailFqg==
+
 "@types/type-is@^1.6.2":
   version "1.6.3"
   resolved "https://registry.yarnpkg.com/@types/type-is/-/type-is-1.6.3.tgz#45285b3be846a4afc9d488910a8e4b7bc2e8a169"